Skip to content

Commit 424b493

Browse files
tyler-dunkeltyler-dunkel
committed
Merge branch 'feature/CG-826' into 'master'
feat(services): add new service dmsReplicationInstance Closes CG-826 See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-aws!216
2 parents 24acbbc + 3a3baea commit 424b493

19 files changed

Lines changed: 543 additions & 40 deletions

‎README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi
8888
| configurationRecorder | iamRole |
8989
| customerGateway | vpnConnection |
9090
| dynamodb | appSync |
91+
| dmsReplicationInstance | securityGroup, subnet, vpc, kms |
9192
| ebs | asg, ec2, emrInstance |
9293
| ec2 | alb, asg, ebs, eip, emrInstance, networkInterface, securityGroup, subnet, vpc, ecsContainer |
9394
| ecr | |
@@ -126,7 +127,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi
126127
| iot | |
127128
| kinesisFirehose | kinesisStream, s3 |
128129
| kinesisStream | kinesisFirehose |
129-
| kms | cloudtrail, cloudwatchLog, codebuild, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, sns, redshiftCluster |
130+
| kms | cloudtrail, cloudwatchLog, codebuild, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, sns, dmsReplicationInstance redshiftCluster |
130131
| lambda | appSync, cognitoUserPool, kms, securityGroup, subnet, vpc |
131132
| managedAirflow | iamRole, securityGroups, subnet, s3 |
132133
| nacl | vpc |
@@ -143,14 +144,14 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi
143144
| sageMakerExperiment | |
144145
| s3 | cloudfront, cloudtrail, kinesisFirehose, managedAirflow |
145146
| secretsManager | |
146-
| securityGroup | alb, asg, clientVpnEndpoint, codebuild, ecsService, lambda, ec2, elasticSearchDomain, elb, rdsCluster, rdsDbInstance, eksCluster, elastiCacheCluster, managedAirflow |
147+
| securityGroup | alb, asg, clientVpnEndpoint, codebuild, dmsReplicationInstance, ecsService, lambda, ec2, elasticSearchDomain, elb, rdsCluster, rdsDbInstance, eksCluster, elastiCacheCluster, managedAirflow |
147148
| ses | |
148149
| sns | kms, cloudtrail, cloudwatch |
149150
| sqs | |
150-
| subnet | alb, asg, codebuild, ec2, ecsService, efsMountTarget, elastiCacheCluster, elasticSearchDomain, elb, lambda, managedAirflow, natGateway, networkInterface, routeTable, vpc, eksCluster, emrCluster, flowLog |
151+
| subnet | alb, asg, codebuild, dmsReplicationInstance, ec2, ecsService, efsMountTarget, elastiCacheCluster, elasticSearchDomain, elb, lambda, managedAirflow, natGateway, networkInterface, routeTable, vpc, eksCluster, emrCluster, flowLog |
151152
| transitGateway | routeTable, transitGatewayAttachment, vpnConnection |
152153
| transitGatewayAttachment | routeTable, transitGateway, vpc, vpnConnection |
153-
| vpc | alb, codebuild, ec2, eip, elb, ecsService, efsMountTarget, eksCluster igw, elastiCacheCluster, elasticSearchDomain, lambda, nacl, natGateway, networkInterface, rdsDbInstance, redshiftCluster, route53HostedZone, routeTable, subnet, flowLog, vpnGateway, transitGatewayAttachment |
154+
| vpc | alb, codebuild, dmsReplicationInstance, ec2, eip, elb, ecsService, efsMountTarget, eksCluster igw, elastiCacheCluster, elasticSearchDomain, lambda, nacl, natGateway, networkInterface, rdsDbInstance, redshiftCluster, route53HostedZone, routeTable, subnet, flowLog, vpnGateway, transitGatewayAttachment |
154155
| vpnConnection | customerGateway, transitGateway, transitGatewayAttachment, vpnGateway |
155156
| vpnGateway | vpc, vpnConnection |
156157
| wafV2WebAcl | |

‎src/enums/schemasMap.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ export default {
2525
[services.cognitoUserPool]: 'awsCognitoUserPool',
2626
[services.configurationRecorder]: 'awsConfigurationRecorder',
2727
[services.customerGateway]: 'awsCustomerGateway',
28+
[services.dmsReplicationInstance]: 'awsDmsReplicationInstance',
2829
[services.dynamodb]: 'awsDynamoDbTable',
2930
[services.ebs]: 'awsEbs',
3031
[services.ec2Instance]: 'awsEc2',

‎src/enums/serviceMap.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ import ManagedAirflow from '../services/managedAirflow'
8888
import WafV2WebAcl from '../services/wafV2WebAcl'
8989
import GuardDutyDetector from '../services/guardDutyDetector'
9090
import ElasticSearchDomain from '../services/elasticSearchDomain'
91+
import DmsReplicationInstance from '../services/dmsReplicationInstance'
9192

9293
/**
9394
* serviceMap is an object that contains all currently supported services for AWS
@@ -134,6 +135,7 @@ export default {
134135
[services.emrCluster]: EmrCluster,
135136
[services.emrInstance]: EmrInstance,
136137
[services.emrStep]: EmrStep,
138+
[services.dmsReplicationInstance]: DmsReplicationInstance,
137139
[services.dynamodb]: DynamoDB,
138140
[services.igw]: AwsInternetGateway,
139141
[services.iot]: IotThingAttribute,

‎src/enums/services.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export default {
2020
cognitoUserPool: 'cognitoUserPool',
2121
configurationRecorder: 'configurationRecorder',
2222
customerGateway: 'customerGateway',
23+
dmsReplicationInstance: 'dmsReplicationInstance',
2324
dynamodb: 'dynamodb',
2425
ebs: 'ebs',
2526
ec2Instance: 'ec2Instance',

‎src/services/dmsReplicationInstance/connections.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { ServiceConnection } from '@cloudgraph/sdk'
2+
3+
import services from '../../enums/services'
4+
import { AwsKms } from '../kms/data'
5+
import { RawAwsDmsReplicationInstance } from '../dmsReplicationInstance/data'
6+
import { AwsSecurityGroup } from '../securityGroup/data'
7+
8+
export default ({
9+
service: replication,
10+
data,
11+
region,
12+
}: {
13+
service: RawAwsDmsReplicationInstance
14+
data: Array<{ name: string; data: { [property: string]: any[] } }>
15+
region: string
16+
}): {
17+
[property: string]: ServiceConnection[]
18+
} => {
19+
const { ReplicationInstanceArn, KmsKeyId, VpcSecurityGroups = [] } = replication
20+
const connections: ServiceConnection[] = []
21+
const sgIds = VpcSecurityGroups.map(({ VpcSecurityGroupId }) => VpcSecurityGroupId)
22+
/**
23+
* Find any kms related data
24+
*/
25+
const keys = data.find(({ name }) => name === services.kms)
26+
if (keys?.data?.[region]) {
27+
const dataAtRegion: AwsKms[] = keys.data[region].filter(
28+
({
29+
Arn
30+
}: AwsKms) => Arn === KmsKeyId
31+
)
32+
for (const key of dataAtRegion) {
33+
connections.push({
34+
id: key.KeyId,
35+
resourceType: services.kms,
36+
relation: 'child',
37+
field: 'kms',
38+
})
39+
}
40+
}
41+
42+
/**
43+
* Find any securityGroups related data
44+
*/
45+
const securityGroups = data.find(({ name }) => name === services.sg)
46+
if (securityGroups?.data?.[region]) {
47+
const dataAtRegion: AwsSecurityGroup[] = securityGroups.data[region].filter(
48+
({
49+
GroupId
50+
}: AwsSecurityGroup) => sgIds.includes(GroupId)
51+
)
52+
for (const sg of dataAtRegion) {
53+
connections.push({
54+
id: sg.GroupId,
55+
resourceType: services.sg,
56+
relation: 'child',
57+
field: 'securityGroups',
58+
})
59+
}
60+
}
61+
62+
const natResult = {
63+
[ReplicationInstanceArn]: connections,
64+
}
65+
return natResult
66+
}

‎src/services/dmsReplicationInstance/data.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import { Config } from 'aws-sdk/lib/config'
2+
import { PromiseResult } from 'aws-sdk/lib/request'
3+
import { AWSError } from 'aws-sdk/lib/error'
4+
import DMS from 'aws-sdk/clients/dms'
5+
import { groupBy, isEmpty } from 'lodash'
6+
import { convertToPromise, fetchAllPaginatedData } from '../../utils/fetchUtils'
7+
import { initTestEndpoint } from '../../utils'
8+
import AwsErrorLog from '../../utils/errorLog'
9+
import { convertAwsTagsToTagMap } from '../../utils/format'
10+
import { TagMap, AwsTag } from '../../types'
11+
12+
const serviceName = 'dms'
13+
const errorLog = new AwsErrorLog(serviceName)
14+
const endpoint = initTestEndpoint(serviceName)
15+
16+
export interface RawAwsDmsReplicationInstance extends DMS.ReplicationInstance {
17+
region: string
18+
Tags: TagMap
19+
}
20+
21+
/**
22+
* DmsReplicationInstance
23+
*/
24+
25+
export default async ({
26+
regions,
27+
config,
28+
}: {
29+
regions: string
30+
config: Config
31+
}): Promise<{ [region: string]: RawAwsDmsReplicationInstance[] }> => {
32+
const result: RawAwsDmsReplicationInstance[] = []
33+
34+
const activeRegions = regions.split(',')
35+
36+
for (const region of activeRegions) {
37+
const client = new DMS({ ...config, region, endpoint })
38+
let dmsReplicationInstanceData: DMS.ReplicationInstance[]
39+
try {
40+
dmsReplicationInstanceData = await fetchAllPaginatedData({
41+
getResourcesFn: convertToPromise({
42+
sdkContext: client,
43+
fnName: 'describeReplicationInstances',
44+
}),
45+
accessor: '',
46+
})
47+
} catch (err) {
48+
errorLog.generateAwsErrorLog({
49+
functionName: 'describeReplicationInstances',
50+
err,
51+
})
52+
}
53+
if (!isEmpty(dmsReplicationInstanceData)) {
54+
const arns = dmsReplicationInstanceData.map(
55+
({ ReplicationInstanceArn }) => ReplicationInstanceArn
56+
)
57+
let tagData: PromiseResult<DMS.ListTagsForResourceResponse, AWSError>
58+
try {
59+
tagData = await client
60+
.listTagsForResource({ ResourceArnList: arns })
61+
.promise()
62+
} catch (err) {
63+
errorLog.generateAwsErrorLog({
64+
functionName: 'listTagsForResource',
65+
err,
66+
})
67+
}
68+
for (const replication of dmsReplicationInstanceData) {
69+
result.push({
70+
...replication,
71+
region,
72+
Tags: convertAwsTagsToTagMap(
73+
tagData?.TagList?.filter(
74+
({ ResourceArn }) =>
75+
ResourceArn === replication.ReplicationInstanceArn
76+
) as AwsTag[]
77+
),
78+
})
79+
}
80+
}
81+
}
82+
83+
errorLog.reset()
84+
return groupBy(result, 'region')
85+
}

‎src/services/dmsReplicationInstance/format.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
import { formatTagsFromMap } from '../../utils/format' // TODO: Build this
2+
import { AwsDmsReplicationInstance } from '../../types/generated'
3+
import { RawAwsDmsReplicationInstance } from './data'
4+
import cuid from 'cuid'
5+
6+
/**
7+
* DmsReplicationInstance
8+
*/
9+
10+
export default ({
11+
account,
12+
service: rawData,
13+
region,
14+
}: {
15+
account: string
16+
service: RawAwsDmsReplicationInstance
17+
region: string
18+
}): AwsDmsReplicationInstance => {
19+
const {
20+
ReplicationInstanceIdentifier: replicationInstanceIdentifier,
21+
ReplicationInstanceArn: arn,
22+
ReplicationInstanceClass: replicationInstanceClass,
23+
ReplicationInstanceStatus: replicationInstanceStatus,
24+
AllocatedStorage: allocatedStorage,
25+
InstanceCreateTime: instanceCreateTime,
26+
VpcSecurityGroups: vpcSecurityGroups,
27+
AvailabilityZone: availabilityZone,
28+
ReplicationSubnetGroup: replicationSubnetGroup,
29+
PreferredMaintenanceWindow: preferredMaintenanceWindow,
30+
PendingModifiedValues: pendingModifiedValues,
31+
MultiAZ: multiAz,
32+
EngineVersion: engineVersion,
33+
AutoMinorVersionUpgrade: autoMinorVersionUpgrade,
34+
KmsKeyId: kmsKeyId,
35+
ReplicationInstancePrivateIpAddress: replicationInstancePrivateIpAddress,
36+
ReplicationInstancePublicIpAddress: replicationInstancePublicIpAddress,
37+
ReplicationInstancePrivateIpAddresses: replicationInstancePrivateIpAddresses,
38+
ReplicationInstancePublicIpAddresses: replicationInstancePublicIpAddresses,
39+
PubliclyAccessible: publiclyAccessible,
40+
SecondaryAvailabilityZone: secondaryAvailabilityZone,
41+
FreeUntil: freeUntil,
42+
DnsNameServers: dnsNameServers,
43+
Tags: tags,
44+
} = rawData
45+
46+
const mappedVpcSecurityGroups = vpcSecurityGroups?.map(({ VpcSecurityGroupId, Status }) => ({
47+
id: cuid(),
48+
status: Status,
49+
vpcSecurityGroupId: VpcSecurityGroupId
50+
}))
51+
52+
const formattedReplicationSubnetGroup = {
53+
replicationSubnetGroupIdentifier: replicationSubnetGroup?.ReplicationSubnetGroupIdentifier,
54+
replicationSubnetGroupDescription: replicationSubnetGroup?.ReplicationSubnetGroupDescription,
55+
vpcId: replicationSubnetGroup?.VpcId,
56+
subnetGroupStatus: replicationSubnetGroup?.SubnetGroupStatus,
57+
subnets: replicationSubnetGroup?.Subnets?.map(({ SubnetAvailabilityZone, SubnetIdentifier, SubnetStatus }) => ({
58+
id: cuid(),
59+
subnetAvailabilityZone: {
60+
name: SubnetAvailabilityZone?.Name
61+
},
62+
subnetIdentifier: SubnetIdentifier,
63+
subnetStatus: SubnetStatus
64+
}))
65+
}
66+
67+
const formattedPendingModifiedValues = {
68+
replicationInstanceClass: pendingModifiedValues?.ReplicationInstanceClass,
69+
allocatedStorage: pendingModifiedValues?.AllocatedStorage,
70+
multiAZ: pendingModifiedValues?.MultiAZ,
71+
engineVersion: pendingModifiedValues?.EngineVersion
72+
}
73+
74+
return {
75+
id: arn,
76+
arn,
77+
region,
78+
accountId: account,
79+
replicationInstanceIdentifier,
80+
replicationInstanceClass,
81+
replicationInstanceStatus,
82+
allocatedStorage,
83+
instanceCreateTime: instanceCreateTime?.toISOString(),
84+
vpcSecurityGroups: mappedVpcSecurityGroups,
85+
availabilityZone,
86+
replicationSubnetGroup: formattedReplicationSubnetGroup,
87+
preferredMaintenanceWindow,
88+
pendingModifiedValues: formattedPendingModifiedValues,
89+
replicationInstancePrivateIpAddress,
90+
replicationInstancePublicIpAddress,
91+
replicationInstancePrivateIpAddresses,
92+
replicationInstancePublicIpAddresses,
93+
multiAz,
94+
engineVersion,
95+
autoMinorVersionUpgrade,
96+
kmsKeyId,
97+
publiclyAccessible,
98+
freeUntil: freeUntil?.toISOString(),
99+
secondaryAvailabilityZone,
100+
dnsNameServers,
101+
tags: formatTagsFromMap(tags ?? {})
102+
}
103+
}

‎src/services/dmsReplicationInstance/index.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { Service } from '@cloudgraph/sdk'
2+
import BaseService from '../base'
3+
import format from './format'
4+
import getData from './data'
5+
import mutation from './mutation'
6+
import getConnections from './connections'
7+
8+
export default class DmsReplicationInstance extends BaseService implements Service {
9+
format = format.bind(this)
10+
11+
getData = getData.bind(this)
12+
13+
getConnections = getConnections.bind(this)
14+
15+
mutation = mutation
16+
}
17+

‎src/services/dmsReplicationInstance/mutation.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
export default `mutation($input: [AddawsDmsReplicationInstanceInput!]!) {
2+
addawsDmsReplicationInstance(input: $input, upsert: true) {
3+
numUids
4+
}
5+
}`;

0 commit comments

Comments
 (0)