Changes in 9dadbfd: bug5838: make creation of auto-scaling group and launch configuration aware of VPCs

				java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java
			
          @@ -1713,7 +1713,7 @@ public class LandscapeServiceImpl implements LandscapeService {

                       .setRegion(region)

                       .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)

                       .setShardingKeys(shardingKeys)

          -            .setReplicaset(applicationReplicaSet)

          +            .setReplicaSet(applicationReplicaSet)

                       .setShardName(shardName)

                       .build()

                       .run();

          @@ -1735,7 +1735,7 @@ public class LandscapeServiceImpl implements LandscapeService {

                       .setRegion(region)

                       .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)

                       .setShardingKeys(shardingkeys)

          -            .setReplicaset(applicationReplicaSet)

          +            .setReplicaSet(applicationReplicaSet)

                       .setShardName(shardName)

                       .build()

                       .run();

          @@ -1768,7 +1768,7 @@ public class LandscapeServiceImpl implements LandscapeService {

                       .setLandscape(getLandscape())

                       .setTargetGroupNamePrefix(LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX)

                       .setShardingKeys(shardingkeys)

          -            .setReplicaset(applicationReplicaSet)

          +            .setReplicaSet(applicationReplicaSet)

                       .setRegion(region)

                       .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)

                       .setShardName(shardName)

				java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java
			
          @@ -341,6 +341,12 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey> 

                   return result;

               }

          +    private Subnet getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(AwsAvailabilityZone az, SecurityGroup securityGroup, Region region) {

          +        final Ec2Client ec2Client = getEc2Client(region);

          +        final String vpcId = ec2Client.describeSecurityGroups(b->b.groupIds(securityGroup.getId())).securityGroups().iterator().next().vpcId();

          +        return ec2Client.describeSubnets(b->b.filters(Filter.builder().name("vpc-id").values(vpcId).build())).subnets().iterator().next();

          +    }

          +    

               private <MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>

               Listener createLoadBalancerHttpListener(ApplicationLoadBalancer<ShardingKey> alb) {

                   return getLoadBalancingClient(getRegion(alb.getRegion()))

          @@ -504,6 +510,11 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey> 

               /**

                * Grabs all subnets that are default subnet for any of the availability zones specified

          +     * <p>

          +     * 

          +     * FIXME bug5838: for a non-default VPC its subnets won't be the default subnets for their AZs either. Hence, we

          +     * need to get the VPC-ID, either immediately or through a security group whose

          +     * {@link software.amazon.awssdk.services.ec2.model.SecurityGroup#vpcId() VPC-ID} could be used

                */

               private Iterable<Subnet> getSubnetsForAvailabilityZones(Region region, Iterable<AwsAvailabilityZone> azs) {

                   return Util.filter(getEc2Client(region).describeSubnets().subnets(), subnet -> subnet.defaultForAz()

          @@ -934,14 +945,8 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey> 

                       .instanceType(instanceType).keyName(keyName)

                       .placement(Placement.builder().availabilityZone(az.getName()).build())

                       .securityGroupIds(Util.mapToArrayList(securityGroups, SecurityGroup::getId));

          -        final List<software.amazon.awssdk.services.ec2.model.SecurityGroup> awsSecurityGroups = ec2Client.describeSecurityGroups(

          -                b->b.groupIds(Util.asList(Util.map(securityGroups, SecurityGroup::getId)))).securityGroups();

          -        ec2Client.describeSubnets().subnets().stream().filter(

          -                subnet->

          -                    subnet.availabilityZoneId().equals(az.getId()) &&

          -                    subnet.vpcId().equals(awsSecurityGroups.iterator().next().vpcId()))

          -                .findFirst()

          -                .map(subnet->runInstancesRequestBuilder.subnetId(subnet.subnetId()));

          +        runInstancesRequestBuilder.subnetId(getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(

          +                az, securityGroups.iterator().next(), getRegion(az.getRegion())).subnetId());

                   if (userData != null) {

                       runInstancesRequestBuilder.userData(Base64.getEncoder().encodeToString(String.join("\n", userData).getBytes()));

                   }

          @@ -1941,18 +1946,20 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey> 

                       String imageId, AwsApplicationConfiguration<ShardingKey, MetricsT, ProcessT> replicaConfiguration,

                       int minReplicas, int maxReplicas, int maxRequestsPerTarget) {

                   logger.info("Creating launch configuration for replica set "+replicaSetName);

          -        final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region));

          +        final Region awsRegion = getRegion(region);

          +        final AutoScalingClient autoScalingClient = getAutoScalingClient(awsRegion);

                   final String releaseName = replicaConfiguration.getRelease().map(r->r.getName()).orElse("UnknownRelease");

                   final String launchConfigurationName = getLaunchConfigurationName(replicaSetName, releaseName);

                   final String autoScalingGroupName = getAutoScalingGroupName(replicaSetName);

                   final Iterable<AwsAvailabilityZone> availabilityZones = getAvailabilityZones(region);

          +        final SecurityGroup securityGroup = getDefaultSecurityGroupForApplicationHosts(region);

                   final int instanceWarmupTimeInSeconds = (int) Duration.ONE_MINUTE.times(3).asSeconds();

                   autoScalingClient.createLaunchConfiguration(b->b

                           .launchConfigurationName(launchConfigurationName)

                           .keyName(keyName)

                           .imageId(imageId)

                           .instanceMonitoring(i->i.enabled(true))

          -                .securityGroups(getDefaultSecurityGroupForApplicationHosts(region).getId())

          +                .securityGroups(securityGroup.getId())

                           .userData(Base64.getEncoder().encodeToString(replicaConfiguration.getAsEnvironmentVariableAssignments().getBytes()))

                           .instanceType(instanceType.toString()));

                   logger.info("Creating auto-scaling group for replica set "+replicaSetName);

          @@ -1962,7 +1969,8 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey> 

                           .maxSize(maxReplicas)

                           .healthCheckGracePeriod(instanceWarmupTimeInSeconds)

                           .autoScalingGroupName(autoScalingGroupName)

          -                .availabilityZones(Util.toArray(Util.map(availabilityZones, az->az.getName()), new String[3]))

          +                .vpcZoneIdentifier(Util.joinStrings(",", Util.map(availabilityZones,

          +                        az->getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(az, securityGroup, awsRegion))))

                           .targetGroupARNs(publicTargetGroup.getTargetGroupArn())

                           .launchConfigurationName(launchConfigurationName);

                       tags.ifPresent(t->{

				java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ShardProcedure.java
			
          @@ -54,8 +54,8 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {

               protected final ShardingKey SHARDING_KEY_UNUSED_BY_ANY_APPLICATION = (ShardingKey) "lauycaluy3cla3yrclaurlIYQL8";

               protected final String shardName;

               final protected Set<ShardingKey> shardingKeys;

          -    final AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;

          -    final Region region;

          +    final protected AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;

          +    final protected Region region;

               private final String pathPrefixForShardingKey;

               protected ShardProcedure(BuilderImpl<?,?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {

          @@ -82,7 +82,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {

                   BuilderT setShardingKeys(Set<ShardingKey> shardingkeys);

          -        BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset);

          +        BuilderT setReplicaSet(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset);

                   BuilderT setRegion(Region region);

               }

          @@ -122,7 +122,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {

                   }

                   @Override

          -        public BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset) {

          +        public BuilderT setReplicaSet(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset) {

                       this.replicaSet = replicaset;

                       return self();

                   }

				wiki/info/landscape/amazon-ec2.md
			
          @@ -194,7 +194,7 @@ In our default region ``eu-west-1`` there are four Amazon Machine Image (AMI) ty

           The SAP Sailing Analytics image is used to launch new instances, shared or dedicated, that host one or more Sailing Analytics application processes. The image contains an installation of the SAP JVM 8 under /opt/sapjvm_8, an Apache httpd service that is not currently used by default for reverse proxying / rewriting / logging activities, an initially empty directory ``/home/sailing/servers`` used to host default application process configurations, and an initialization script under ``/etc/init.d/sailing`` that handles the instance's initialization with a default application process from the EC2 instance's user data. Instructions for setting up such an image from scratch can be found [here](/wiki/info/landscape/creating-ec2-image-from-scratch).

          -The user data line ``image-upgrade`` will cause the image to ignore all application configuration data and only bring the new instance to an updated state. For this, the Git content under ``/home/sailing/code`` is brought to the latest master branch commit, a ``yum update`` is carried out to install all operating system package updates available, log directories and the ``/home/sailing/servers`` directory are cleared, and the ``root`` user's crontab is brought up to date from the Git ``configuration/crontab`` file. If the ``no-shutdown`` line is provided in the instance's user data, the instance will be left running. Otherwise, it will shut down which would be a good default for creating a new image. See also [Upgrading AMIs](#amazon-ec2-for-sap-sailing-analytics_automated-procedures_upgrading-amis) for procedures that automate much of this upgrade process.

          +The user data line ``image-upgrade`` will cause the image to ignore all application configuration data and only bring the new instance to an updated state. For this, the Git content under ``/home/sailing/code`` is brought to the latest master branch commit, a ``yum update`` is carried out to install all operating system package updates available, log directories and the ``/home/sailing/servers`` directory are cleared, and the ``root`` user's crontab is brought up to date from the Git ``configuration/crontab`` file. If the ``no-shutdown`` line is provided in the instance's user data, the instance will be left running. Otherwise, it will shut down which would be a good default for creating a new image. See also  procedures that automate much of this upgrade process.

           The MongoDB Live Replica Set NVMe image is used to scale out or upgrade existing MongoDB replica sets. It also reads the EC2 instance's user data during start-up and can be parameterized by the following variables: ``REPLICA_SET_NAME``, ``REPLICA_SET_PRIMARY``, ``REPLICA_SET_PRIORITY``, and ``REPLICA_SET_VOTES``. An example configuration could look like this:

           ```

...	...	@@ -1713,7 +1713,7 @@ public class LandscapeServiceImpl implements LandscapeService {
1713	1713	.setRegion(region)
1714	1714	.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
1715	1715	.setShardingKeys(shardingKeys)
1716		- .setReplicaset(applicationReplicaSet)
	1716	+ .setReplicaSet(applicationReplicaSet)
1717	1717	.setShardName(shardName)
1718	1718	.build()
1719	1719	.run();
...	...	@@ -1735,7 +1735,7 @@ public class LandscapeServiceImpl implements LandscapeService {
1735	1735	.setRegion(region)
1736	1736	.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
1737	1737	.setShardingKeys(shardingkeys)
1738		- .setReplicaset(applicationReplicaSet)
	1738	+ .setReplicaSet(applicationReplicaSet)
1739	1739	.setShardName(shardName)
1740	1740	.build()
1741	1741	.run();
...	...	@@ -1768,7 +1768,7 @@ public class LandscapeServiceImpl implements LandscapeService {
1768	1768	.setLandscape(getLandscape())
1769	1769	.setTargetGroupNamePrefix(LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX)
1770	1770	.setShardingKeys(shardingkeys)
1771		- .setReplicaset(applicationReplicaSet)
	1771	+ .setReplicaSet(applicationReplicaSet)
1772	1772	.setRegion(region)
1773	1773	.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
1774	1774	.setShardName(shardName)

...	...	@@ -341,6 +341,12 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
341	341	return result;
342	342	}
343	343
	344	+ private Subnet getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(AwsAvailabilityZone az, SecurityGroup securityGroup, Region region) {
	345	+ final Ec2Client ec2Client = getEc2Client(region);
	346	+ final String vpcId = ec2Client.describeSecurityGroups(b->b.groupIds(securityGroup.getId())).securityGroups().iterator().next().vpcId();
	347	+ return ec2Client.describeSubnets(b->b.filters(Filter.builder().name("vpc-id").values(vpcId).build())).subnets().iterator().next();
	348	+ }
	349	+
344	350	private <MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
345	351	Listener createLoadBalancerHttpListener(ApplicationLoadBalancer<ShardingKey> alb) {
346	352	return getLoadBalancingClient(getRegion(alb.getRegion()))
...	...	@@ -504,6 +510,11 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
504	510
505	511	/**
506	512	* Grabs all subnets that are default subnet for any of the availability zones specified
	513	+ * <p>
	514	+ *
	515	+ * FIXME bug5838: for a non-default VPC its subnets won't be the default subnets for their AZs either. Hence, we
	516	+ * need to get the VPC-ID, either immediately or through a security group whose
	517	+ * {@link software.amazon.awssdk.services.ec2.model.SecurityGroup#vpcId() VPC-ID} could be used
507	518	*/
508	519	private Iterable<Subnet> getSubnetsForAvailabilityZones(Region region, Iterable<AwsAvailabilityZone> azs) {
509	520	return Util.filter(getEc2Client(region).describeSubnets().subnets(), subnet -> subnet.defaultForAz()
...	...	@@ -934,14 +945,8 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
934	945	.instanceType(instanceType).keyName(keyName)
935	946	.placement(Placement.builder().availabilityZone(az.getName()).build())
936	947	.securityGroupIds(Util.mapToArrayList(securityGroups, SecurityGroup::getId));
937		- final List<software.amazon.awssdk.services.ec2.model.SecurityGroup> awsSecurityGroups = ec2Client.describeSecurityGroups(
938		- b->b.groupIds(Util.asList(Util.map(securityGroups, SecurityGroup::getId)))).securityGroups();
939		- ec2Client.describeSubnets().subnets().stream().filter(
940		- subnet->
941		- subnet.availabilityZoneId().equals(az.getId()) &&
942		- subnet.vpcId().equals(awsSecurityGroups.iterator().next().vpcId()))
943		- .findFirst()
944		- .map(subnet->runInstancesRequestBuilder.subnetId(subnet.subnetId()));
	948	+ runInstancesRequestBuilder.subnetId(getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(
	949	+ az, securityGroups.iterator().next(), getRegion(az.getRegion())).subnetId());
945	950	if (userData != null) {
946	951	runInstancesRequestBuilder.userData(Base64.getEncoder().encodeToString(String.join("\n", userData).getBytes()));
947	952	}
...	...	@@ -1941,18 +1946,20 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
1941	1946	String imageId, AwsApplicationConfiguration<ShardingKey, MetricsT, ProcessT> replicaConfiguration,
1942	1947	int minReplicas, int maxReplicas, int maxRequestsPerTarget) {
1943	1948	logger.info("Creating launch configuration for replica set "+replicaSetName);
1944		- final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region));
	1949	+ final Region awsRegion = getRegion(region);
	1950	+ final AutoScalingClient autoScalingClient = getAutoScalingClient(awsRegion);
1945	1951	final String releaseName = replicaConfiguration.getRelease().map(r->r.getName()).orElse("UnknownRelease");
1946	1952	final String launchConfigurationName = getLaunchConfigurationName(replicaSetName, releaseName);
1947	1953	final String autoScalingGroupName = getAutoScalingGroupName(replicaSetName);
1948	1954	final Iterable<AwsAvailabilityZone> availabilityZones = getAvailabilityZones(region);
	1955	+ final SecurityGroup securityGroup = getDefaultSecurityGroupForApplicationHosts(region);
1949	1956	final int instanceWarmupTimeInSeconds = (int) Duration.ONE_MINUTE.times(3).asSeconds();
1950	1957	autoScalingClient.createLaunchConfiguration(b->b
1951	1958	.launchConfigurationName(launchConfigurationName)
1952	1959	.keyName(keyName)
1953	1960	.imageId(imageId)
1954	1961	.instanceMonitoring(i->i.enabled(true))
1955		- .securityGroups(getDefaultSecurityGroupForApplicationHosts(region).getId())
	1962	+ .securityGroups(securityGroup.getId())
1956	1963	.userData(Base64.getEncoder().encodeToString(replicaConfiguration.getAsEnvironmentVariableAssignments().getBytes()))
1957	1964	.instanceType(instanceType.toString()));
1958	1965	logger.info("Creating auto-scaling group for replica set "+replicaSetName);
...	...	@@ -1962,7 +1969,8 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
1962	1969	.maxSize(maxReplicas)
1963	1970	.healthCheckGracePeriod(instanceWarmupTimeInSeconds)
1964	1971	.autoScalingGroupName(autoScalingGroupName)
1965		- .availabilityZones(Util.toArray(Util.map(availabilityZones, az->az.getName()), new String[3]))
	1972	+ .vpcZoneIdentifier(Util.joinStrings(",", Util.map(availabilityZones,
	1973	+ az->getSubnetForAvailabilityZoneInSameVpcAsSecurityGroup(az, securityGroup, awsRegion))))
1966	1974	.targetGroupARNs(publicTargetGroup.getTargetGroupArn())
1967	1975	.launchConfigurationName(launchConfigurationName);
1968	1976	tags.ifPresent(t->{

...	...	@@ -54,8 +54,8 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
54	54	protected final ShardingKey SHARDING_KEY_UNUSED_BY_ANY_APPLICATION = (ShardingKey) "lauycaluy3cla3yrclaurlIYQL8";
55	55	protected final String shardName;
56	56	final protected Set<ShardingKey> shardingKeys;
57		- final AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;
58		- final Region region;
	57	+ final protected AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;
	58	+ final protected Region region;
59	59	private final String pathPrefixForShardingKey;
60	60
61	61	protected ShardProcedure(BuilderImpl<?,?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {
...	...	@@ -82,7 +82,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
82	82
83	83	BuilderT setShardingKeys(Set<ShardingKey> shardingkeys);
84	84
85		- BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset);
	85	+ BuilderT setReplicaSet(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset);
86	86
87	87	BuilderT setRegion(Region region);
88	88	}
...	...	@@ -122,7 +122,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
122	122	}
123	123
124	124	@Override
125		- public BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset) {
	125	+ public BuilderT setReplicaSet(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset) {
126	126	this.replicaSet = replicaset;
127	127	return self();
128	128	}

...	...	@@ -194,7 +194,7 @@ In our default region ``eu-west-1`` there are four Amazon Machine Image (AMI) ty
194	194
195	195	The SAP Sailing Analytics image is used to launch new instances, shared or dedicated, that host one or more Sailing Analytics application processes. The image contains an installation of the SAP JVM 8 under /opt/sapjvm_8, an Apache httpd service that is not currently used by default for reverse proxying / rewriting / logging activities, an initially empty directory ``/home/sailing/servers`` used to host default application process configurations, and an initialization script under ``/etc/init.d/sailing`` that handles the instance's initialization with a default application process from the EC2 instance's user data. Instructions for setting up such an image from scratch can be found [here](/wiki/info/landscape/creating-ec2-image-from-scratch).
196	196
197		-The user data line ``image-upgrade`` will cause the image to ignore all application configuration data and only bring the new instance to an updated state. For this, the Git content under ``/home/sailing/code`` is brought to the latest master branch commit, a ``yum update`` is carried out to install all operating system package updates available, log directories and the ``/home/sailing/servers`` directory are cleared, and the ``root`` user's crontab is brought up to date from the Git ``configuration/crontab`` file. If the ``no-shutdown`` line is provided in the instance's user data, the instance will be left running. Otherwise, it will shut down which would be a good default for creating a new image. See also [Upgrading AMIs](#amazon-ec2-for-sap-sailing-analytics_automated-procedures_upgrading-amis) for procedures that automate much of this upgrade process.
	197	+The user data line ``image-upgrade`` will cause the image to ignore all application configuration data and only bring the new instance to an updated state. For this, the Git content under ``/home/sailing/code`` is brought to the latest master branch commit, a ``yum update`` is carried out to install all operating system package updates available, log directories and the ``/home/sailing/servers`` directory are cleared, and the ``root`` user's crontab is brought up to date from the Git ``configuration/crontab`` file. If the ``no-shutdown`` line is provided in the instance's user data, the instance will be left running. Otherwise, it will shut down which would be a good default for creating a new image. See also procedures that automate much of this upgrade process.
198	198
199	199	The MongoDB Live Replica Set NVMe image is used to scale out or upgrade existing MongoDB replica sets. It also reads the EC2 instance's user data during start-up and can be parameterized by the following variables: ``REPLICA_SET_NAME``, ``REPLICA_SET_PRIMARY``, ``REPLICA_SET_PRIORITY``, and ``REPLICA_SET_VOTES``. An example configuration could look like this:
200	200	```