aws_route53: delete_existing is creating and deleting 2x

[BwL1289]

Describe the bug

It's possible there's some user error in here.

When delete_existing=True, and something goes awry from another resource related to the A record (see below for reproduction steps), I am seeing the custom resource run twice so the a record gets created, then deleted, then created, and then finally deleted so by the time the stack finishes updating, there's no A record at all.

Here are the logs:

INIT_START Runtime Version: nodejs:20.v57	Runtime Version ARN: arn:aws:lambda:<redacted>>::runtime:9d084cce5cc7578c503eb8fe4bf7891c94c8f5f0ccb036f3f8c3a01cf5212db6
START RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41 Version: $LATEST
2025-04-23T13:17:12.072Z	2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	INFO	
{
    "RequestType": "Create",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "49412ef1-b119-4959-a442-db1326a97e04",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:17:16.254Z	2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425%7C49412ef1-b119-4959-a442-db1326a97e04?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '49412ef1-b119-4959-a442-db1326a97e04',
  PhysicalResourceId: '49412ef1-b119-4959-a442-db1326a97e04',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41
REPORT RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	Duration: 4390.45 ms	Billed Duration: 4391 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	Init Duration: 160.14 ms	
START RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd Version: $LATEST
2025-04-23T13:17:23.497Z	00d5ccf0-8492-45c6-aef7-8841946360dd	INFO	
{
    "RequestType": "Delete",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "c8e33677-92bd-4fe3-bff6-379b2d73532e",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170",
    "PhysicalResourceId": "7d099ac6-c8b9-475b-bc4a-84680622fac6",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:17:23.498Z	00d5ccf0-8492-45c6-aef7-8841946360dd	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170%7Cc8e33677-92bd-4fe3-bff6-379b2d73532e?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: 'c8e33677-92bd-4fe3-bff6-379b2d73532e',
  PhysicalResourceId: '7d099ac6-c8b9-475b-bc4a-84680622fac6',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd
REPORT RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd	Duration: 120.10 ms	Billed Duration: 121 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	
START RequestId: 882d23c0-226a-4461-881f-5306de3602f8 Version: $LATEST
2025-04-23T13:22:42.452Z	882d23c0-226a-4461-881f-5306de3602f8	INFO	
{
    "RequestType": "Create",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "5ee90893-a5c8-40b0-a989-90c0bb925043",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:22:42.772Z	882d23c0-226a-4461-881f-5306de3602f8	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170%7C5ee90893-a5c8-40b0-a989-90c0bb925043?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '5ee90893-a5c8-40b0-a989-90c0bb925043',
  PhysicalResourceId: '5ee90893-a5c8-40b0-a989-90c0bb925043',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 882d23c0-226a-4461-881f-5306de3602f8
REPORT RequestId: 882d23c0-226a-4461-881f-5306de3602f8	Duration: 620.24 ms	Billed Duration: 621 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	
START RequestId: 43437399-5250-4621-adf4-6845830879a4 Version: $LATEST
2025-04-23T13:23:53.667Z	43437399-5250-4621-adf4-6845830879a4	INFO	
{
    "RequestType": "Delete",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "1a791d51-f495-445e-a66b-4b1a934e05dd",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425",
    "PhysicalResourceId": "49412ef1-b119-4959-a442-db1326a97e04",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:23:53.731Z	43437399-5250-4621-adf4-6845830879a4	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425%7C1a791d51-f495-445e-a66b-4b1a934e05dd?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '1a791d51-f495-445e-a66b-4b1a934e05dd',
  PhysicalResourceId: '49412ef1-b119-4959-a442-db1326a97e04',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425',
  NoEcho: undefined,
  Data: undefined
}

Regression Issue

• Select this option if this issue appears to be a regression.

Last Known Working CDK Version

No response

Expected Behavior

The existing a record gets deleted and then created with the new value.

Current Behavior

By the time the stack finishes updating, there's no A record at all.

Reproduction Steps

1. Deploy with:
   • An A record with delete_existing=True
   • An ALB w/ protocol=lb.ApplicationProtocol.HTTP
2. Attempt to move ALB to protocol=lb.ApplicationProtocol.HTTPS which may require domain_name and domain_zone to be set. In my case there was a mistake in here somewhere, and the domain_name got set to some other value (i.e. original value was api.dev.example.com an the new domain_name gets set to just dev.example.com), the A record was deleted and recreated with the new record name.
3. Try to force Cfn to recreate the record set by changing the logicalid of the A record
4. Experience error

Possible Solution

Detect if the a record has already been deleted and (re)created by the custom resource and if so, don't run the CR again.

Additional Information/Context

No response

CDK CLI Version

2.1000.3 (build 321a46a)

Framework Version

No response

Node.js Version

v22.12.0

OS

Mac

Language

Python

Language Version

No response

Other information

No response

[ykethan]

appears to be related to #26754

[ykethan]

Hey @BwL1289, thank you for reporting this issue and providing us the logs. On diving into this issue, I was able to observe the following when reproducing this issue.

CDK versions

 "aws-cdk-lib": "2.191.0",
"aws-cdk": "^2.1012.0"

Reproduction Steps:

1. Created a CDK stack with a Route53 PublicHostedZone and an ARecord:

export class Route53IssueStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    // Step 1: Create a VPC
    const vpc = new ec2.Vpc(this, 'VPC', {
      maxAzs: 2,
      natGateways: 0,
    });

    // Step 1: Create an ALB
    const alb = new elbv2.ApplicationLoadBalancer(this, 'ALB', {
      vpc,
      internetFacing: true,
    });

    // Create a hosted zone
    const hostedZone = new route53.PublicHostedZone(this, 'HostedZone', {
      zoneName: 'mydemo-domain.net',
    });

    // Create an A record
    new route53.ARecord(this, 'ARecord', {
      zone: hostedZone,
      recordName: 'api.dev.mydemo-domain.net',
      target: route53.RecordTarget.fromAlias(
        new targets.LoadBalancerTarget(alb)
      ),
      deleteExisting: true,
    });

    // Add HTTP listener
    alb.addListener('HttpListener', {
      port: 80,
      protocol: elbv2.ApplicationProtocol.HTTP,
      defaultAction: elbv2.ListenerAction.fixedResponse(200, {
        contentType: 'text/plain',
        messageBody: 'OK',
      }),
    });
  }
}

2. ran cdk synth && cdk deploy, the deploy was succesfull
3. Updated zoneName and recordName:

    // Create a hosted zone
    const hostedZone = new route53.PublicHostedZone(this, 'HostedZone', {
      zoneName: 'mydemo-domain-abc.net',
    });

    // Create an A record
    new route53.ARecord(this, 'ARecord', {
      zone: hostedZone,
      recordName: 'api.dev.mydemo-domain-abc.net',
      target: route53.RecordTarget.fromAlias(
        new targets.LoadBalancerTarget(alb)
      ),
      deleteExisting: true,
    });

4. ran cdk synth

HostedZoneDB99F866:
  Type: AWS::Route53::HostedZone
  Properties:
    Name: mydemo-domain.net.  # Should be mydemo-domain-abc.net.

5. Tried clearing the CDK context with cdk context --clear and re-synthesizing, the issue persists.
6. On deploy observed duplicate record events in the logs

Marking this as P1

[BwL1289]

@ykethan thank you

[ykethan]

Hey @BwL1289, wanted to follow up on this. Tried reproing this issue with code from the previous comment and the following but no longer observe the duplicate operations. Could you let us know are still experiencing this and provide us a minimal reproduction code?

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2';
import * as route53 from 'aws-cdk-lib/aws-route53';
import * as targets from 'aws-cdk-lib/aws-route53-targets';
import * as customResources from 'aws-cdk-lib/custom-resources';

export class Route53IssueStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    // Step 1: Create a VPC
    const vpc = new ec2.Vpc(this, 'VPC', {
      maxAzs: 2,
      natGateways: 0,
    });

    // Step 1: Create an ALB
    const alb = new elbv2.ApplicationLoadBalancer(this, 'ALB', {
      vpc,
      internetFacing: true,
    });

    // Create a hosted zone
    const hostedZone = new route53.PublicHostedZone(this, 'HostedZone', {
      zoneName: 'mydemo-domain-dev.net',
    });

    // Create an A record
    new route53.ARecord(this, 'ARecord1', {
      zone: hostedZone,
      recordName: 'api.dev.mydemo-domain.net',
      target: route53.RecordTarget.fromAlias(
        new targets.LoadBalancerTarget(alb),
      ),
      deleteExisting: true,
    });

    // Add HTTP listener
    alb.addListener('HttpListener', {
      port: 80,
      protocol: elbv2.ApplicationProtocol.HTTPS,
      certificates: [
        elbv2.ListenerCertificate.fromArn(
          <Arn>,
        ),
      ],
      defaultAction: elbv2.ListenerAction.fixedResponse(200, {
        contentType: 'text/plain',
        messageBody: 'OK',
      }),
    });
  }
}

[BwL1289]

@ykethan i can't try reproducing it again as I'm blocked due to #34290 I opened (which is in turn related upstream to #34235).

[BwL1289]

@ykethan this is still a bug and I just experienced it. This should be prioritized because 1) it just took down production for us 2) I am now worried about deploying to production again 3) I had to actually patch this in the console.