java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java
... ...
@@ -83,7 +83,6 @@ import com.sap.sse.landscape.aws.AwsInstance;
83 83
import com.sap.sse.landscape.aws.AwsLandscape;
84 84
import com.sap.sse.landscape.aws.AwsShard;
85 85
import com.sap.sse.landscape.aws.HostSupplier;
86
-import com.sap.sse.landscape.aws.LandscapeConstants;
87 86
import com.sap.sse.landscape.aws.ReverseProxy;
88 87
import com.sap.sse.landscape.aws.ReverseProxyCluster;
89 88
import com.sap.sse.landscape.aws.Tags;
... ...
@@ -287,19 +286,18 @@ public class LandscapeServiceImpl implements LandscapeService {
287 286
sendMailAboutNewArchiveCandidate(replicaSet);
288 287
final ScheduledExecutorService monitorTaskExecutor = ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor();
289 288
final ArchiveCandidateMonitoringBackgroundTask monitoringTask = new ArchiveCandidateMonitoringBackgroundTask(
290
- getSecurityService().getCurrentUser(), this, replicaSet, candidateHostname, monitorTaskExecutor, bearerTokenUsedByReplicas);
289
+ getSecurityService().getCurrentUser(), this, replicaSet, candidateHostname, monitorTaskExecutor,
290
+ bearerTokenUsedByReplicas);
291 291
monitorTaskExecutor.execute(monitoringTask);
292 292
}
293 293
294 294
private AwsAvailabilityZone getBestAvailabilityZoneForArchiveCandidate(AwsRegion region, AwsLandscape<String> landscape,
295 295
ReverseProxyCluster<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, RotatingFileBasedLog> reverseProxyCluster,
296 296
String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {
297
- final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> oldArchiveReplicaSet = getLandscape()
298
- .getApplicationReplicaSetByTagValue(region,
299
- SharedLandscapeConstants.SAILING_ANALYTICS_APPLICATION_HOST_TAG,
300
- SharedLandscapeConstants.ARCHIVE_SERVER_APPLICATION_REPLICA_SET_NAME,
301
- new SailingAnalyticsHostSupplier<String>(), Landscape.WAIT_FOR_PROCESS_TIMEOUT,
302
- Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
297
+ final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> oldArchiveReplicaSet = getApplicationReplicaSet(
298
+ region, SharedLandscapeConstants.ARCHIVE_SERVER_APPLICATION_REPLICA_SET_NAME,
299
+ Landscape.WAIT_FOR_PROCESS_TIMEOUT.map(Duration::asMillis).orElse(null),
300
+ optionalKeyName, privateKeyEncryptionPassphrase);
303 301
final SailingAnalyticsProcess<String> oldArchivePrimary = oldArchiveReplicaSet.getMaster();
304 302
final AwsAvailabilityZone oldArchiveAZ = oldArchivePrimary.getHost().getAvailabilityZone();
305 303
AwsAvailabilityZone result = null;
... ...
@@ -307,6 +305,9 @@ public class LandscapeServiceImpl implements LandscapeService {
307 305
final AwsAvailabilityZone az = reverseProxyHost.getAvailabilityZone();
308 306
if (!az.equals(oldArchiveAZ)) {
309 307
result = az;
308
+ logger.info("Identified availability zone (AZ) " + result
309
+ + " for new ARCHIVE server; it differs from current ARCHIVE's AZ " + oldArchiveAZ
310
+ + " and has reverse proxy host " + reverseProxyHost.getInstanceId() + " in that AZ.");
310 311
break;
311 312
}
312 313
}
... ...
@@ -328,7 +329,9 @@ public class LandscapeServiceImpl implements LandscapeService {
328 329
}
329 330
330 331
@Override
331
- public void makeCandidateArchiveServerGoLive(String regionId, String optionalKeyName, byte[] privateKeyEncryptionPassphrase, String optionalDomainName) throws Exception {
332
+ public void makeCandidateArchiveServerGoLive(String regionId, String optionalKeyName,
333
+ byte[] privateKeyEncryptionPassphrase, String optionalDomainName)
334
+ throws Exception {
332 335
final AwsLandscape<String> landscape = getLandscape();
333 336
final AwsRegion region = new AwsRegion(regionId, landscape);
334 337
final String candidateHostname = getHostname(SharedLandscapeConstants.ARCHIVE_CANDIDATE_SUBDOMAIN, optionalDomainName);
... ...
@@ -337,15 +340,28 @@ public class LandscapeServiceImpl implements LandscapeService {
337 340
Landscape.WAIT_FOR_PROCESS_TIMEOUT.map(Duration::asMillis).orElse(null),
338 341
optionalKeyName, privateKeyEncryptionPassphrase);
339 342
if (archiveReplicaSet == null) {
340
- throw new IllegalArgumentException("Couldn't find candidate replica set with name "+SharedLandscapeConstants.ARCHIVE_SERVER_APPLICATION_REPLICA_SET_NAME+" in region "+regionId);
343
+ throw new IllegalArgumentException("Couldn't find candidate replica set with name "
344
+ + SharedLandscapeConstants.ARCHIVE_SERVER_APPLICATION_REPLICA_SET_NAME + " in region " + regionId);
341 345
}
342
- // TODO bug6203: update 000-macros.conf so that internal IP of candidate becomes ARCHIVE_IP, and ARCHIVE_IP becomes ARCHIVE_FAILOVER_IP, and maybe a backup copy of ARCHIVE_FAILOVER_COPY is kept as a comment
343
- // TODO bug6203: then refresh the configuration...
344
- int TODO;
346
+ final SailingAnalyticsProcess<String> candidate = archiveReplicaSet.getMaster();
345 347
final ReverseProxy<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, RotatingFileBasedLog> reverseProxyCluster =
346 348
getLandscape().getCentralReverseProxy(region);
349
+ final Pair<String, String> archiveAndFailoverIPs = reverseProxyCluster
350
+ .getArchiveAndFailoverIPs(Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
351
+ logger.info("Found new candidate " + candidate.getHost()
352
+ .getInstanceId() + " with internal IP "
353
+ + candidate.getHost().getPrivateAddress() + " and current production ARCHIVE "
354
+ + archiveAndFailoverIPs.getA() + ". Turning production into failover and candidate into production.");
355
+ reverseProxyCluster.setArchiveAndFailoverIPs(candidate.getHost().getPrivateAddress().getHostAddress(),
356
+ archiveAndFailoverIPs.getA(), Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
357
+ final SailingAnalyticsHost<String> oldFailover = getLandscape().getHostByPrivateDnsNameOrIpAddress(region,
358
+ archiveAndFailoverIPs.getB(), new SailingAnalyticsHostSupplier<>());
359
+ logger.info("Terminating old failover host " + oldFailover.getInstanceId() + " with internal IP "
360
+ + oldFailover.getPrivateAddress());
361
+ oldFailover.terminate();
347 362
logger.info("Removing reverse proxy rule for archive candidate with hostname "+ candidateHostname);
348 363
reverseProxyCluster.removeRedirect(candidateHostname, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
364
+ sendMailAboutNewArchiveServerLive(archiveReplicaSet);
349 365
}
350 366
351 367
@Override
... ...
@@ -1864,10 +1880,9 @@ public class LandscapeServiceImpl implements LandscapeService {
1864 1880
* This is to be called after the rotation of candidate/production/failover ARCHIVE has happend, to inform
1865 1881
*/
1866 1882
private void sendMailAboutNewArchiveServerLive(
1867
- AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet,
1868
- Optional<Action> alsoSendToAllUsersWithThisPermissionOnReplicaSet) throws MailException {
1883
+ AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet) throws MailException {
1869 1884
sendMailToReplicaSetOwner(replicaSet, "NewArchiveServerLiveSubject", "NewArchiveServerLiveBody",
1870
- alsoSendToAllUsersWithThisPermissionOnReplicaSet);
1885
+ Optional.of(ServerActions.CONFIGURE_LOCAL_SERVER));
1871 1886
}
1872 1887
1873 1888
/**
java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/ReverseProxy.java
... ...
@@ -2,6 +2,8 @@ package com.sap.sse.landscape.aws;
2 2
3 3
import java.util.Optional;
4 4
import java.util.UUID;
5
+
6
+import com.sap.sse.common.Util.Pair;
5 7
import com.sap.sse.landscape.Host;
6 8
import com.sap.sse.landscape.Log;
7 9
import com.sap.sse.landscape.application.ApplicationProcess;
... ...
@@ -124,4 +126,12 @@ public interface ReverseProxy<ShardingKey, MetricsT extends ApplicationProcessMe
124 126
* member instances and their availability zones.
125 127
*/
126 128
String getTargetGroupHealthCheckPath(String targetGroupArn);
129
+
130
+ /**
131
+ * Fetches the ARCHIVE_IP and ARCHIVE_FAILOVER_IP definitions from the 000-macros.conf file in the reverse proxy and returns
132
+ * them, in this order, as strings.
133
+ */
134
+ Pair<String, String> getArchiveAndFailoverIPs(Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception;
135
+
136
+ void setArchiveAndFailoverIPs(String hostAddress, String b, Optional<String> ofNullable, byte[] privateKeyEncryptionPassphrase);
127 137
}
java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApacheReverseProxy.java
... ...
@@ -6,7 +6,9 @@ import java.util.Optional;
6 6
import java.util.UUID;
7 7
import java.util.logging.Level;
8 8
import java.util.logging.Logger;
9
+
9 10
import com.sap.sse.common.Duration;
11
+import com.sap.sse.common.Util.Pair;
10 12
import com.sap.sse.landscape.Host;
11 13
import com.sap.sse.landscape.RotatingFileBasedLog;
12 14
import com.sap.sse.landscape.application.ApplicationProcess;
... ...
@@ -76,6 +78,20 @@ implements com.sap.sse.landscape.Process<RotatingFileBasedLog, MetricsT> {
76 78
private static final String HOME_ARCHIVE_REDIRECT_MACRO = "Home-ARCHIVE";
77 79
private static final String EVENT_ARCHIVE_REDIRECT_MACRO = "Event-ARCHIVE";
78 80
private static final String SERIES_ARCHIVE_REDIRECT_MACRO = "Series-ARCHIVE";
81
+ private static final String CONFIG_FILE_FOR_ARCHIVE_AND_FAILOVER_DEFINITION = "000-macros"+CONFIG_FILE_EXTENSION;
82
+
83
+ /**
84
+ * Name of the "macro"/variable definition used in the file identified by {@link #CONFIG_FILE_FOR_ARCHIVE_AND_FAILOVER_DEFINITION}
85
+ * that specifies the internal IP address of the primary ARCHIVE server to use.
86
+ */
87
+ private static final String ARCHIVE_IP = "ARCHIVE_IP";
88
+
89
+ /**
90
+ * Name of the "macro"/variable definition used in the file identified by {@link #CONFIG_FILE_FOR_ARCHIVE_AND_FAILOVER_DEFINITION}
91
+ * that specifies the internal IP address of the fail-over ARCHIVE server to use.
92
+ */
93
+ private static final String ARCHIVE_FAILOVER_IP = "ARCHIVE_FAILOVER_IP";
94
+
79 95
private final AwsInstance<ShardingKey> host;
80 96
81 97
public ApacheReverseProxy(AwsLandscape<ShardingKey> landscape, AwsInstance<ShardingKey> host) {
... ...
@@ -101,10 +117,27 @@ implements com.sap.sse.landscape.Process<RotatingFileBasedLog, MetricsT> {
101 117
*/
102 118
public void rotateLogs(Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {
103 119
final String command = "logrotate --force -v /etc/logrotate.d/httpd 2>&1; echo \"logrotate done\"";
104
- logger.info("Standard output from forced log rotate on " + this.getHostname() + ": " + runCommandAndReturnStdoutAndStderr(command, "Standard error from logrotate ",
120
+ logger.info("Standard output from forced log rotate on " + this.getHostname() + ": " + runCommandAndReturnStdoutAndLogStderr(command, "Standard error from logrotate ",
105 121
Level.ALL, optionalKeyName, privateKeyEncryptionPassphrase));
106 122
}
107 123
124
+ @Override
125
+ public Pair<String, String> getArchiveAndFailoverIPs(Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {
126
+ final String command = "cat "+CONFIG_REPO_PATH+"/"+RELATIVE_CONFIG_PATH+"/"+CONFIG_FILE_FOR_ARCHIVE_AND_FAILOVER_DEFINITION+" | grep \"^Define "+ARCHIVE_IP+"\" | sed -e 's/^Define "+ARCHIVE_IP+" //'; "
127
+ + "cat "+CONFIG_REPO_PATH+"/"+RELATIVE_CONFIG_PATH+"/"+CONFIG_FILE_FOR_ARCHIVE_AND_FAILOVER_DEFINITION+" | grep \"^Define "+ARCHIVE_FAILOVER_IP+"\" | sed -e 's/^Define "+ARCHIVE_FAILOVER_IP+" //'";
128
+ final String[] archiveAndFailoverIPs = runCommandAndReturnStdoutAndLogStderr(command,
129
+ "Standard error from getting "+ARCHIVE_IP+" and "+ARCHIVE_FAILOVER_IP+": ",
130
+ Level.INFO, optionalKeyName, privateKeyEncryptionPassphrase).split("\n");
131
+ return new Pair<>(archiveAndFailoverIPs[0], archiveAndFailoverIPs[1]);
132
+ }
133
+
134
+ @Override
135
+ public void setArchiveAndFailoverIPs(String hostAddress, String b, Optional<String> ofNullable,
136
+ byte[] privateKeyEncryptionPassphrase) {
137
+ // TODO Auto-generated method stub
138
+
139
+ }
140
+
108 141
/**
109 142
* Creates a redirect file and updates the git repo.
110 143
*
... ...
@@ -139,26 +172,26 @@ implements com.sap.sse.landscape.Process<RotatingFileBasedLog, MetricsT> {
139 172
command = command + "'; service httpd reload"; // Concludes the su. And reloads as the root user.
140 173
logger.info("Standard output from setting up the re-direct for " + hostname
141 174
+ " and reloading the Apache httpd server: "
142
- + runCommandAndReturnStdoutAndStderr(command,
175
+ + runCommandAndReturnStdoutAndLogStderr(command,
143 176
"Standard error from setting up the re-direct for " + hostname
144 177
+ " and reloading the Apache httpd server: ",
145 178
Level.INFO, optionalKeyName, privateKeyEncryptionPassphrase));
146 179
}
147
-
180
+
148 181
/**
149 182
* Overloads {@link #setRedirect(String, String, String, Optional, byte[], boolean, boolean, String...)} and
150 183
* defaults to {@code true} and {@code true} for committing and pushing.
151 184
*
152 185
* @see #setRedirect(String, String, String, Optional, byte[], boolean, boolean, String...)
153 186
*/
154
- public void setRedirect(String configFileNameForHostname, String macroName, String hostname,
187
+ private void setRedirect(String configFileNameForHostname, String macroName, String hostname,
155 188
Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase, String... macroArguments)
156 189
throws Exception {
157 190
setRedirect(configFileNameForHostname, macroName, hostname, optionalKeyName, privateKeyEncryptionPassphrase,
158 191
/* doCommit */ true, /* doPush */ true, macroArguments);
159 192
}
160 193
161
- private String runCommandAndReturnStdoutAndStderr(String command, String stderrLogPrefix, Level stderrLogLevel,
194
+ private String runCommandAndReturnStdoutAndLogStderr(String command, String stderrLogPrefix, Level stderrLogLevel,
162 195
Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {
163 196
final SshCommandChannel sshChannel = getHost().createRootSshChannel(TIMEOUT, optionalKeyName, privateKeyEncryptionPassphrase);
164 197
final String stdout = sshChannel.runCommandAndReturnStdoutAndLogStderr(command, stderrLogPrefix, stderrLogLevel);
... ...
@@ -282,7 +315,7 @@ implements com.sap.sse.landscape.Process<RotatingFileBasedLog, MetricsT> {
282 315
command.append("'; service httpd reload;"); // ' closes the su. The reload must be run as the root user.
283 316
logger.info("Standard output from removing the re-direct for " + hostname
284 317
+ " and reloading the Apache httpd server: "
285
- + runCommandAndReturnStdoutAndStderr(command.toString(),
318
+ + runCommandAndReturnStdoutAndLogStderr(command.toString(),
286 319
"Standard error from removing the re-direct for " + hostname
287 320
+ " and reloading the Apache httpd server: ",
288 321
Level.INFO, optionalKeyName, privateKeyEncryptionPassphrase));
java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApacheReverseProxyCluster.java
... ...
@@ -14,6 +14,7 @@ import java.util.logging.Logger;
14 14
import com.sap.sse.landscape.aws.LandscapeConstants;
15 15
import com.sap.sse.common.Duration;
16 16
import com.sap.sse.common.Util;
17
+import com.sap.sse.common.Util.Pair;
17 18
import com.sap.sse.concurrent.ConsumerWithException;
18 19
import com.sap.sse.landscape.Landscape;
19 20
import com.sap.sse.landscape.Log;
... ...
@@ -252,4 +253,10 @@ public class ApacheReverseProxyCluster<ShardingKey, MetricsT extends Application
252 253
byte[] privateKeyEncryptionPassphrase) throws Exception {
253 254
setRedirect(proxy -> proxy.removeRedirect(scope, optionalKeyName, privateKeyEncryptionPassphrase));
254 255
}
256
+
257
+ @Override
258
+ public Pair<String, String> getArchiveAndFailoverIPs(Optional<String> optionalKeyName,
259
+ byte[] privateKeyEncryptionPassphrase) throws Exception {
260
+ return getReverseProxies().iterator().next().getArchiveAndFailoverIPs(optionalKeyName, privateKeyEncryptionPassphrase);
261
+ }
255 262
}