f21dec9337d88c0b76a7b01371865b935eba8283
java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/LandscapeService.java
| ... | ... | @@ -17,8 +17,10 @@ import com.sap.sailing.landscape.procedures.SailingProcessConfigurationVariables |
| 17 | 17 | import com.sap.sailing.landscape.procedures.StartMultiServer; |
| 18 | 18 | import com.sap.sailing.server.gateway.interfaces.CompareServersResult; |
| 19 | 19 | import com.sap.sailing.server.gateway.interfaces.SailingServer; |
| 20 | +import com.sap.sailing.server.gateway.interfaces.SailingServerFactory; |
|
| 20 | 21 | import com.sap.sse.common.Duration; |
| 21 | 22 | import com.sap.sse.common.Util.Triple; |
| 23 | +import com.sap.sse.common.mail.MailException; |
|
| 22 | 24 | import com.sap.sse.landscape.Release; |
| 23 | 25 | import com.sap.sse.landscape.application.ApplicationReplicaSet; |
| 24 | 26 | import com.sap.sse.landscape.aws.AmazonMachineImage; |
| ... | ... | @@ -29,6 +31,8 @@ import com.sap.sse.landscape.aws.impl.AwsRegion; |
| 29 | 31 | import com.sap.sse.landscape.mongodb.Database; |
| 30 | 32 | import com.sap.sse.landscape.mongodb.MongoEndpoint; |
| 31 | 33 | import com.sap.sse.security.SecurityService; |
| 34 | +import com.sap.sse.security.shared.HasPermissions.Action; |
|
| 35 | +import com.sap.sse.security.shared.impl.User; |
|
| 32 | 36 | |
| 33 | 37 | import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup; |
| 34 | 38 | import software.amazon.awssdk.services.ec2.model.InstanceType; |
| ... | ... | @@ -492,4 +496,27 @@ public interface LandscapeService { |
| 492 | 496 | String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception; |
| 493 | 497 | |
| 494 | 498 | String getHostname(String replicaSetName, String optionalDomainName); |
| 499 | + |
|
| 500 | + /** |
|
| 501 | + * @param subjectMessageKey |
|
| 502 | + * must have a single placeholder argument representing the name of the replica set |
|
| 503 | + * @param bodyMessageKey |
|
| 504 | + * must have a single placeholder argument representing the name of the replica set |
|
| 505 | + * @param alsoSendToAllUsersWithThisPermissionOnReplicaSet |
|
| 506 | + * when not empty, all users that have permission to this {@link SecuredSecurityTypes#SERVER SERVER} |
|
| 507 | + * action on the {@code replicaSet} will receive the e-mail in addition to the server owner. No user |
|
| 508 | + * will receive the e-mail twice. |
|
| 509 | + */ |
|
| 510 | + void sendMailToReplicaSetOwner( |
|
| 511 | + AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet, |
|
| 512 | + String subjectMessageKey, String bodyMessageKey, |
|
| 513 | + Optional<Action> alsoSendToAllUsersWithThisPermissionOnReplicaSet) throws MailException; |
|
| 514 | + |
|
| 515 | + void sendMailToCurrentUser(String messageSubjectKey, String messageBodyKey, String... messageParameters) |
|
| 516 | + throws MailException; |
|
| 517 | + |
|
| 518 | + void sendMailToUser(User user, String messageSubjectKey, String messageBodyKey, String... messageParameters) |
|
| 519 | + throws MailException; |
|
| 520 | + |
|
| 521 | + SailingServerFactory getSailingServerFactory(); |
|
| 495 | 522 | } |
java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/ArchiveCandidateMonitoringBackgroundTask.java
| ... | ... | @@ -1,27 +1,28 @@ |
| 1 | 1 | package com.sap.sailing.landscape.impl; |
| 2 | 2 | |
| 3 | +import java.net.URL; |
|
| 3 | 4 | import java.util.Arrays; |
| 4 | 5 | import java.util.Iterator; |
| 5 | -import java.util.LinkedList; |
|
| 6 | -import java.util.List; |
|
| 7 | 6 | import java.util.Optional; |
| 8 | 7 | import java.util.concurrent.ScheduledExecutorService; |
| 9 | 8 | import java.util.concurrent.TimeUnit; |
| 10 | 9 | import java.util.logging.Logger; |
| 11 | 10 | |
| 12 | -import org.apache.shiro.subject.Subject; |
|
| 13 | - |
|
| 11 | +import com.sap.sailing.landscape.LandscapeService; |
|
| 14 | 12 | import com.sap.sailing.landscape.SailingAnalyticsMetrics; |
| 15 | 13 | import com.sap.sailing.landscape.SailingAnalyticsProcess; |
| 14 | +import com.sap.sailing.server.gateway.interfaces.CompareServersResult; |
|
| 15 | +import com.sap.sailing.server.gateway.interfaces.SailingServer; |
|
| 16 | 16 | import com.sap.sse.common.Duration; |
| 17 | 17 | import com.sap.sse.common.Named; |
| 18 | 18 | import com.sap.sse.common.TimePoint; |
| 19 | 19 | import com.sap.sse.common.impl.NamedImpl; |
| 20 | +import com.sap.sse.common.mail.MailException; |
|
| 20 | 21 | import com.sap.sse.landscape.Landscape; |
| 21 | 22 | import com.sap.sse.landscape.RotatingFileBasedLog; |
| 22 | 23 | import com.sap.sse.landscape.aws.AwsApplicationReplicaSet; |
| 23 | -import com.sap.sse.landscape.aws.AwsLandscape; |
|
| 24 | 24 | import com.sap.sse.landscape.aws.ReverseProxy; |
| 25 | +import com.sap.sse.security.shared.impl.User; |
|
| 25 | 26 | |
| 26 | 27 | /** |
| 27 | 28 | * A stateful monitoring task that can be {@link #run run} to observe an {@code ARCHIVE} candidate process and wait for |
| ... | ... | @@ -52,8 +53,10 @@ import com.sap.sse.landscape.aws.ReverseProxy; |
| 52 | 53 | public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 53 | 54 | private interface Check extends Named { |
| 54 | 55 | boolean runCheck() throws Exception; |
| 56 | + void setLastFailureMessage(String lastFailureMessage); |
|
| 55 | 57 | boolean hasTimedOut(); |
| 56 | 58 | Duration getDelayAfterFailure(); |
| 59 | + String getLastFailureMessage(); |
|
| 57 | 60 | } |
| 58 | 61 | |
| 59 | 62 | private abstract class AbstractCheck extends NamedImpl implements Check { |
| ... | ... | @@ -61,6 +64,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 61 | 64 | private final TimePoint creationTime; |
| 62 | 65 | private final Duration timeout; |
| 63 | 66 | private final Duration delayAfterFailure; |
| 67 | + private String lastFailureMessage; |
|
| 64 | 68 | |
| 65 | 69 | public AbstractCheck(String name, Duration timeout, Duration delayAfterFailure) { |
| 66 | 70 | super(name); |
| ... | ... | @@ -78,6 +82,16 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 78 | 82 | public Duration getDelayAfterFailure() { |
| 79 | 83 | return delayAfterFailure; |
| 80 | 84 | } |
| 85 | + |
|
| 86 | + @Override |
|
| 87 | + public String getLastFailureMessage() { |
|
| 88 | + return lastFailureMessage; |
|
| 89 | + } |
|
| 90 | + |
|
| 91 | + @Override |
|
| 92 | + public void setLastFailureMessage(String lastFailureMessage) { |
|
| 93 | + this.lastFailureMessage = lastFailureMessage; |
|
| 94 | + } |
|
| 81 | 95 | } |
| 82 | 96 | |
| 83 | 97 | private static final Logger logger = Logger.getLogger(ArchiveCandidateMonitoringBackgroundTask.class.getName()); |
| ... | ... | @@ -87,40 +101,56 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 87 | 101 | private final static double MAXIMUM_ONE_MINUTE_SYSTEM_LOAD_AVERAGE = 2.0; |
| 88 | 102 | private final static int MAXIMUM_THREAD_POOL_QUEUE_SIZE = 10; |
| 89 | 103 | private final static Optional<Duration> TIMEOUT_FIRST_CONTACT = Optional.of(Landscape.WAIT_FOR_PROCESS_TIMEOUT.get().plus(Landscape.WAIT_FOR_HOST_TIMEOUT.get())); |
| 90 | - private final Subject subject; |
|
| 91 | - private final AwsLandscape<String> landscape; |
|
| 104 | + private final static Duration SERVER_COMPARISON_TIMEOUT = Duration.ONE_MINUTE.times(10); // good for two or three attempts, usually |
|
| 105 | + private final static Duration DELAY_BETWEEN_COMPARISON_CHECKS = Duration.ONE_MINUTE; |
|
| 106 | + |
|
| 107 | + /** |
|
| 108 | + * The user on whose behalf the monitoring is performed; this is used for sending notifications about the monitoring |
|
| 109 | + * result to the user and for performing the monitoring with the same permissions as the user (e.g. when accessing |
|
| 110 | + * the candidate's REST API) |
|
| 111 | + */ |
|
| 112 | + private final User currentUser; |
|
| 113 | + private final String candidateHostname; |
|
| 114 | + private final LandscapeService landscapeService; |
|
| 92 | 115 | private final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet; |
| 93 | 116 | private final ReverseProxy<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, RotatingFileBasedLog> reverseProxyCluster; |
| 94 | 117 | private final String optionalKeyName; |
| 95 | 118 | private final byte[] privateKeyEncryptionPassphrase; |
| 96 | 119 | private final ScheduledExecutorService executor; |
| 97 | - private final TimePoint firstRun; |
|
| 98 | - private final List<String> messagesToSendToProcessOwner; |
|
| 120 | + |
|
| 121 | + /** |
|
| 122 | + * A bearer token expected to authenticate the {@link #currentUser} against the candidate and production ARCHIVE |
|
| 123 | + * servers |
|
| 124 | + */ |
|
| 125 | + private final String effectiveBearerToken; |
|
| 126 | + |
|
| 99 | 127 | private Iterable<Check> checks; |
| 100 | 128 | private Iterator<Check> checksIterator; |
| 101 | 129 | private Check currentCheck; |
| 130 | + |
|
| 131 | + |
|
| 102 | 132 | |
| 103 | - public ArchiveCandidateMonitoringBackgroundTask(Subject subject, AwsLandscape<String> landscape, |
|
| 133 | + public ArchiveCandidateMonitoringBackgroundTask(User currentUser, LandscapeService landscapeService, |
|
| 104 | 134 | AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet, |
| 105 | 135 | String candidateHostname, |
| 106 | 136 | ReverseProxy<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, RotatingFileBasedLog> reverseProxyCluster, |
| 107 | - String optionalKeyName, byte[] privateKeyEncryptionPassphrase, ScheduledExecutorService executor) { |
|
| 108 | - this.subject = subject; |
|
| 109 | - this.landscape = landscape; |
|
| 137 | + String optionalKeyName, |
|
| 138 | + byte[] privateKeyEncryptionPassphrase, ScheduledExecutorService executor, String effectiveBearerToken) { |
|
| 139 | + this.currentUser = currentUser; |
|
| 140 | + this.landscapeService = landscapeService; |
|
| 110 | 141 | this.replicaSet = replicaSet; |
| 142 | + this.candidateHostname = candidateHostname; |
|
| 111 | 143 | this.reverseProxyCluster = reverseProxyCluster; |
| 112 | 144 | this.optionalKeyName = optionalKeyName; |
| 113 | 145 | this.privateKeyEncryptionPassphrase = privateKeyEncryptionPassphrase; |
| 114 | 146 | this.executor = executor; |
| 115 | - this.firstRun = TimePoint.now(); |
|
| 116 | - this.messagesToSendToProcessOwner = new LinkedList<>(); |
|
| 147 | + this.effectiveBearerToken = effectiveBearerToken; |
|
| 117 | 148 | this.checks = Arrays.asList( |
| 118 | 149 | new IsReady(), |
| 119 | 150 | new HasLowEnoughSystemLoad(), |
| 120 | 151 | new HasShortEnoughDefaultBackgroundThreadPoolExecutorQueue(), |
| 121 | 152 | new HasShortEnoughDefaultForegroundThreadPoolExecutorQueue(), |
| 122 | - new CompareServersWithRestAPI(), |
|
| 123 | - new CompareServersByLeaderboardGroups()); |
|
| 153 | + new CompareServersWithRestAPI()); |
|
| 124 | 154 | this.checksIterator = this.checks.iterator(); |
| 125 | 155 | this.currentCheck = checksIterator.next(); |
| 126 | 156 | } |
| ... | ... | @@ -137,19 +167,27 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 137 | 167 | // re-schedule this task to run next check immediately |
| 138 | 168 | executor.submit(this); |
| 139 | 169 | } else { |
| 140 | - logger.info("Done with all checks; candidate is ready for production."); |
|
| 141 | 170 | // all checks passed; candidate is ready for production; nothing more to do here |
| 171 | + logger.info("Done with all checks; candidate is ready for production."); |
|
| 172 | + notifyProcessOwnerCandidateIsReadyForProduction(); // this ends the re-scheduling loop |
|
| 142 | 173 | } |
| 143 | 174 | } else { |
| 144 | 175 | rescheduleCurrentCheckAfterFailureOrTimeout(); |
| 145 | 176 | } |
| 146 | 177 | } catch (Exception e) { |
| 147 | 178 | logger.warning("Exception while running check " + currentCheck + " for candidate " + replicaSet.getMaster().getHost().getHostname() + ": " + e.getMessage()); |
| 179 | + currentCheck.setLastFailureMessage(e.getMessage()); |
|
| 148 | 180 | } |
| 149 | 181 | } |
| 150 | 182 | |
| 151 | - private void rescheduleCurrentCheckAfterFailureOrTimeout() { |
|
| 152 | - executor.schedule(this, currentCheck.getDelayAfterFailure().asMillis(), TimeUnit.MILLISECONDS); |
|
| 183 | + private void rescheduleCurrentCheckAfterFailureOrTimeout() throws MailException { |
|
| 184 | + if (currentCheck.hasTimedOut()) { |
|
| 185 | + logger.severe("Check "+currentCheck+" failed and has timed out; giving up on candidate "+replicaSet.getMaster().getHost().getHostname()); |
|
| 186 | + notifyProcessOwnerCandidateFailedToBecomeReadyForProduction(); // this ends the re-scheduling loop |
|
| 187 | + } else { |
|
| 188 | + logger.info("Check "+currentCheck+" failed but has not yet timed out; re-scheduling to check again after "+currentCheck.getDelayAfterFailure()); |
|
| 189 | + executor.schedule(this, currentCheck.getDelayAfterFailure().asMillis(), TimeUnit.MILLISECONDS); |
|
| 190 | + } |
|
| 153 | 191 | } |
| 154 | 192 | |
| 155 | 193 | private class IsReady extends AbstractCheck { |
| ... | ... | @@ -161,7 +199,11 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 161 | 199 | |
| 162 | 200 | @Override |
| 163 | 201 | public boolean runCheck() throws Exception { |
| 164 | - return replicaSet.getMaster().isReady(Landscape.WAIT_FOR_PROCESS_TIMEOUT); |
|
| 202 | + final boolean result = replicaSet.getMaster().isReady(Landscape.WAIT_FOR_PROCESS_TIMEOUT); |
|
| 203 | + if (!result) { |
|
| 204 | + setLastFailureMessage("Candidate is not ready yet"); |
|
| 205 | + } |
|
| 206 | + return result; |
|
| 165 | 207 | } |
| 166 | 208 | } |
| 167 | 209 | |
| ... | ... | @@ -174,9 +216,14 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 174 | 216 | |
| 175 | 217 | @Override |
| 176 | 218 | public boolean runCheck() throws Exception { |
| 177 | - return replicaSet.getMaster().getLastMinuteSystemLoadAverage(Landscape.WAIT_FOR_PROCESS_TIMEOUT) < MAXIMUM_ONE_MINUTE_SYSTEM_LOAD_AVERAGE; |
|
| 219 | + final double lastMinuteSystemLoadAverage = replicaSet.getMaster().getLastMinuteSystemLoadAverage(Landscape.WAIT_FOR_PROCESS_TIMEOUT); |
|
| 220 | + final boolean result = lastMinuteSystemLoadAverage < MAXIMUM_ONE_MINUTE_SYSTEM_LOAD_AVERAGE; |
|
| 221 | + if (!result) { |
|
| 222 | + setLastFailureMessage("Candidate has too high system load average of "+lastMinuteSystemLoadAverage+ |
|
| 223 | + " which is still above the maximum of "+MAXIMUM_ONE_MINUTE_SYSTEM_LOAD_AVERAGE); |
|
| 224 | + } |
|
| 225 | + return result; |
|
| 178 | 226 | } |
| 179 | - |
|
| 180 | 227 | } |
| 181 | 228 | |
| 182 | 229 | private class HasShortEnoughDefaultBackgroundThreadPoolExecutorQueue extends AbstractCheck { |
| ... | ... | @@ -188,7 +235,13 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 188 | 235 | |
| 189 | 236 | @Override |
| 190 | 237 | public boolean runCheck() throws Exception { |
| 191 | - return replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT) < MAXIMUM_THREAD_POOL_QUEUE_SIZE; |
|
| 238 | + final int defaultBackgroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT); |
|
| 239 | + final boolean result = defaultBackgroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE; |
|
| 240 | + if (!result) { |
|
| 241 | + setLastFailureMessage("Candidate has too many tasks in default background thread pool executor queue: "+defaultBackgroundThreadPoolExecutorQueueSize+ |
|
| 242 | + " which is still above the maximum of "+MAXIMUM_THREAD_POOL_QUEUE_SIZE); |
|
| 243 | + } |
|
| 244 | + return result; |
|
| 192 | 245 | } |
| 193 | 246 | } |
| 194 | 247 | |
| ... | ... | @@ -201,7 +254,13 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 201 | 254 | |
| 202 | 255 | @Override |
| 203 | 256 | public boolean runCheck() throws Exception { |
| 204 | - return replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT) < MAXIMUM_THREAD_POOL_QUEUE_SIZE; |
|
| 257 | + final int defaultForegroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT); |
|
| 258 | + final boolean result = defaultForegroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE; |
|
| 259 | + if (!result) { |
|
| 260 | + setLastFailureMessage("Candidate has too many tasks in default foreground thread pool executor queue: "+defaultForegroundThreadPoolExecutorQueueSize+ |
|
| 261 | + " which is still above the maximum of "+MAXIMUM_THREAD_POOL_QUEUE_SIZE); |
|
| 262 | + } |
|
| 263 | + return result; |
|
| 205 | 264 | } |
| 206 | 265 | } |
| 207 | 266 | |
| ... | ... | @@ -209,28 +268,37 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable { |
| 209 | 268 | private static final long serialVersionUID = -5271988056894947109L; |
| 210 | 269 | |
| 211 | 270 | public CompareServersWithRestAPI() { |
| 212 | - super("compare servers with REST API", LONG_TIMEOUT, DELAY_BETWEEN_CHECKS); |
|
| 271 | + super("compare servers with REST API", SERVER_COMPARISON_TIMEOUT, DELAY_BETWEEN_COMPARISON_CHECKS); |
|
| 213 | 272 | } |
| 214 | 273 | |
| 215 | - |
|
| 216 | 274 | @Override |
| 217 | 275 | public boolean runCheck() throws Exception { |
| 218 | - // TODO Auto-generated method stub |
|
| 219 | - return false; |
|
| 276 | + final SailingServer productionServer = landscapeService.getSailingServerFactory().getSailingServer(new URL("https", replicaSet.getHostname(), "/"), effectiveBearerToken); |
|
| 277 | + final SailingServer candidateServer = landscapeService.getSailingServerFactory().getSailingServer(new URL("https", candidateHostname, "/"), effectiveBearerToken); |
|
| 278 | + final CompareServersResult comparisonResult = candidateServer.compareServers(Optional.empty(), productionServer, Optional.empty()); |
|
| 279 | + if (comparisonResult.hasDiffs()) { |
|
| 280 | + setLastFailureMessage( |
|
| 281 | + "Candidate server does not match production server according to REST API comparison." |
|
| 282 | + + "\nDifferences on candidate side: " + comparisonResult.getADiffs() |
|
| 283 | + + "\nDifferences on production side: " + comparisonResult.getBDiffs() |
|
| 284 | + + "\nNot proceeding further. You need to resolve the issues manually."); // TODO add link to running REST API comparison in browser |
|
| 285 | + } |
|
| 286 | + return !comparisonResult.hasDiffs(); |
|
| 220 | 287 | } |
| 221 | 288 | } |
| 222 | 289 | |
| 223 | - private class CompareServersByLeaderboardGroups extends AbstractCheck { |
|
| 224 | - private static final long serialVersionUID = -5271988056894947109L; |
|
| 290 | + private void notifyProcessOwnerCandidateFailedToBecomeReadyForProduction() throws MailException { |
|
| 291 | + landscapeService.sendMailToUser(currentUser, "NewArchiveCandidateFailedSubject", |
|
| 292 | + "NewArchiveCandidateFailedBody", replicaSet.getServerName(), currentCheck.getName(), |
|
| 293 | + currentCheck.getLastFailureMessage()); |
|
| 294 | + } |
|
| 225 | 295 | |
| 226 | - public CompareServersByLeaderboardGroups() { |
|
| 227 | - super("compare servers with Leaderboard Groups", LONG_TIMEOUT, DELAY_BETWEEN_CHECKS); |
|
| 228 | - } |
|
| 296 | + private void notifyProcessOwnerCandidateIsReadyForProduction() throws MailException { |
|
| 297 | + // TODO send a mail to the process owner that the candidate is ready for production, comparisons were OK, remaining is an optional spot-checke (human in the loop). Include links for spot-checking and triggering the rotation |
|
| 298 | + } |
|
| 229 | 299 | |
| 230 | - @Override |
|
| 231 | - public boolean runCheck() throws Exception { |
|
| 232 | - // TODO Auto-generated method stub |
|
| 233 | - return false; |
|
| 234 | - } |
|
| 300 | + private void sendMailAboutNewArchiveCandidate() throws MailException { |
|
| 301 | + landscapeService.sendMailToUser(currentUser, "StartingNewArchiveCandidateSubject", "StartingNewArchiveCandidateBody", replicaSet.getServerName()); |
|
| 302 | + landscapeService.sendMailToReplicaSetOwner(replicaSet, "RefrainFromArchivingSubject", "RefrainFromArchivingBody", Optional.empty()); |
|
| 235 | 303 | } |
| 236 | 304 | } |
java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java
| ... | ... | @@ -25,7 +25,6 @@ import java.util.logging.Level; |
| 25 | 25 | import java.util.logging.Logger; |
| 26 | 26 | |
| 27 | 27 | import org.apache.http.client.ClientProtocolException; |
| 28 | -import org.apache.shiro.SecurityUtils; |
|
| 29 | 28 | import org.apache.shiro.authz.AuthorizationException; |
| 30 | 29 | import org.json.simple.parser.ParseException; |
| 31 | 30 | import org.osgi.framework.BundleContext; |
| ... | ... | @@ -134,7 +133,7 @@ import software.amazon.awssdk.services.sts.model.Credentials; |
| 134 | 133 | public class LandscapeServiceImpl implements LandscapeService { |
| 135 | 134 | private static final Logger logger = Logger.getLogger(LandscapeServiceImpl.class.getName()); |
| 136 | 135 | |
| 137 | - private static final String STRING_MESSAGES_BASE_NAME = "stringmessages/SailingLandscape_StringMessages"; |
|
| 136 | + public static final String STRING_MESSAGES_BASE_NAME = "stringmessages/SailingLandscape_StringMessages"; |
|
| 138 | 137 | |
| 139 | 138 | private static final String TEMPORARY_UPGRADE_REPLICA_NAME_SUFFIX = " (Upgrade Replica)"; |
| 140 | 139 | |
| ... | ... | @@ -243,12 +242,13 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 243 | 242 | String optionalKeyName, byte[] privateKeyEncryptionPassphrase, String securityServiceReplicationBearerToken, String replicaReplicationBearerToken, |
| 244 | 243 | String optionalDomainName, Integer optionalMemoryInMegabytesOrNull, |
| 245 | 244 | Integer optionalMemoryTotalSizeFactorOrNull, Integer optionalIgtimiRiotPort) throws Exception { |
| 245 | + assert getSecurityService().getCurrentUser() != null; |
|
| 246 | 246 | final AwsLandscape<String> landscape = getLandscape(); |
| 247 | - final String hostname = getHostname(SharedLandscapeConstants.ARCHIVE_CANDIDATE_SUBDOMAIN, optionalDomainName); |
|
| 248 | - final Iterable<ResourceRecordSet> existingDNSRulesForHostname = landscape.getResourceRecordSets(hostname); |
|
| 247 | + final String candidateHostname = getHostname(SharedLandscapeConstants.ARCHIVE_CANDIDATE_SUBDOMAIN, optionalDomainName); |
|
| 248 | + final Iterable<ResourceRecordSet> existingDNSRulesForHostname = landscape.getResourceRecordSets(candidateHostname); |
|
| 249 | 249 | // Failing early in case DNS record already exists (see also bug 5826): |
| 250 | 250 | if (existingDNSRulesForHostname != null && !Util.isEmpty(existingDNSRulesForHostname)) { |
| 251 | - throw new IllegalArgumentException("DNS record for "+hostname+" already exists"); |
|
| 251 | + throw new IllegalArgumentException("DNS record for "+candidateHostname+" already exists"); |
|
| 252 | 252 | } |
| 253 | 253 | final AwsRegion region = new AwsRegion(regionId, landscape); |
| 254 | 254 | final Release release = getRelease(releaseNameOrNullForLatestMaster); |
| ... | ... | @@ -281,13 +281,13 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 281 | 281 | final ReverseProxy<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, RotatingFileBasedLog> reverseProxyCluster = |
| 282 | 282 | getLandscape().getCentralReverseProxy(region); |
| 283 | 283 | final String privateIpAdress = master.getHost().getPrivateAddress().getHostAddress(); |
| 284 | - logger.info("Adding reverse proxy rule for archive candidate with hostname "+ hostname + " and private ip address " + privateIpAdress); |
|
| 285 | - reverseProxyCluster.setPlainRedirect(hostname, master, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); |
|
| 284 | + logger.info("Adding reverse proxy rule for archive candidate with hostname "+ candidateHostname + " and private ip address " + privateIpAdress); |
|
| 285 | + reverseProxyCluster.setPlainRedirect(candidateHostname, master, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); |
|
| 286 | 286 | sendMailAboutNewArchiveCandidate(replicaSet); |
| 287 | 287 | final ScheduledExecutorService monitorTaskExecutor = ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor(); |
| 288 | 288 | final ArchiveCandidateMonitoringBackgroundTask monitoringTask = new ArchiveCandidateMonitoringBackgroundTask( |
| 289 | - SecurityUtils.getSubject(), getLandscape(), replicaSet, hostname, reverseProxyCluster, optionalKeyName, |
|
| 290 | - privateKeyEncryptionPassphrase, monitorTaskExecutor); |
|
| 289 | + getSecurityService().getCurrentUser(), this, replicaSet, candidateHostname, reverseProxyCluster, optionalKeyName, |
|
| 290 | + privateKeyEncryptionPassphrase, monitorTaskExecutor, bearerTokenUsedByReplicas); |
|
| 291 | 291 | monitorTaskExecutor.execute(monitoringTask); |
| 292 | 292 | return replicaSet; |
| 293 | 293 | } |
| ... | ... | @@ -1740,22 +1740,28 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 1740 | 1740 | |
| 1741 | 1741 | private void sendMailAboutNewArchiveCandidate( |
| 1742 | 1742 | AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet) throws MailException { |
| 1743 | - final ResourceBundleStringMessages stringMessages = ResourceBundleStringMessages.create(STRING_MESSAGES_BASE_NAME, getClass().getClassLoader(), StandardCharsets.UTF_8.name()); |
|
| 1743 | + sendMailToCurrentUser("StartingNewArchiveCandidateSubject", "StartingNewArchiveCandidateBody", replicaSet.getServerName()); |
|
| 1744 | + sendMailToReplicaSetOwner(replicaSet, "RefrainFromArchivingSubject", "RefrainFromArchivingBody", Optional.empty()); |
|
| 1745 | + } |
|
| 1746 | + |
|
| 1747 | + @Override |
|
| 1748 | + public void sendMailToCurrentUser(String messageSubjectKey, String messageBodyKey, String... messageParameters) throws MailException { |
|
| 1744 | 1749 | final User currentUser = getSecurityService().getCurrentUser(); |
| 1745 | - if (currentUser != null && currentUser.isEmailValidated()) { |
|
| 1746 | - final String subject = stringMessages.get(currentUser.getLocaleOrDefault(), "StartingNewArchiveCandidateSubject", replicaSet.getServerName()); |
|
| 1747 | - final String body = stringMessages.get(currentUser.getLocaleOrDefault(), "StartingNewArchiveCandidateBody", replicaSet.getServerName()); |
|
| 1748 | - getSecurityService().sendMail(currentUser.getName(), subject, body); |
|
| 1750 | + sendMailToUser(currentUser, messageSubjectKey, messageBodyKey, messageParameters); |
|
| 1751 | + } |
|
| 1752 | + |
|
| 1753 | + @Override |
|
| 1754 | + public void sendMailToUser(final User user, String messageSubjectKey, String messageBodyKey, |
|
| 1755 | + String... messageParameters) throws MailException { |
|
| 1756 | + final ResourceBundleStringMessages stringMessages = ResourceBundleStringMessages.create(STRING_MESSAGES_BASE_NAME, getClass().getClassLoader(), StandardCharsets.UTF_8.name()); |
|
| 1757 | + if (user != null && user.isEmailValidated()) { |
|
| 1758 | + final String subject = stringMessages.get(user.getLocaleOrDefault(), messageSubjectKey, messageParameters); |
|
| 1759 | + final String body = stringMessages.get(user.getLocaleOrDefault(), messageBodyKey, messageParameters); |
|
| 1760 | + getSecurityService().sendMail(user.getName(), subject, body); |
|
| 1749 | 1761 | } else { |
| 1750 | 1762 | logger.warning("Not sending e-mail about new archive candidate to current user because no user is logged in or email address of logged in user "+ |
| 1751 | - (currentUser == null ? "" : currentUser.getName()+" ")+"is not validated"); |
|
| 1763 | + (user == null ? "" : user.getName()+" ")+"is not validated"); |
|
| 1752 | 1764 | } |
| 1753 | - sendMailToReplicaSetOwner(replicaSet, "RefrainFromArchivingSubject", "RefrainFromArchivingBody", Optional.empty()); |
|
| 1754 | - } |
|
| 1755 | - |
|
| 1756 | - private void sendMailAboutNewArchiveServerLive( |
|
| 1757 | - AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet) throws MailException { |
|
| 1758 | - sendMailToReplicaSetOwner(replicaSet, "NewArchiveServerLiveSubject", "NewArchiveServerLiveBody", Optional.empty()); |
|
| 1759 | 1765 | } |
| 1760 | 1766 | |
| 1761 | 1767 | private void sendMailAboutMasterUnavailable( |
| ... | ... | @@ -1778,7 +1784,8 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 1778 | 1784 | * action on the {@code replicaSet} will receive the e-mail in addition to the server owner. No user |
| 1779 | 1785 | * will receive the e-mail twice. |
| 1780 | 1786 | */ |
| 1781 | - private void sendMailToReplicaSetOwner( |
|
| 1787 | + @Override |
|
| 1788 | + public void sendMailToReplicaSetOwner( |
|
| 1782 | 1789 | AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet, |
| 1783 | 1790 | final String subjectMessageKey, final String bodyMessageKey, Optional<Action> alsoSendToAllUsersWithThisPermissionOnReplicaSet) throws MailException { |
| 1784 | 1791 | final Iterable<User> usersToSendMailTo = getSecurityService().getUsersToInformAboutReplicaSet(replicaSet.getServerName(), alsoSendToAllUsersWithThisPermissionOnReplicaSet); |
| ... | ... | @@ -1796,6 +1803,16 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 1796 | 1803 | } |
| 1797 | 1804 | } |
| 1798 | 1805 | } |
| 1806 | + |
|
| 1807 | + /** |
|
| 1808 | + * This is to be called after the rotation of candidate/production/failover ARCHIVE has happend, to inform |
|
| 1809 | + */ |
|
| 1810 | + private void sendMailAboutNewArchiveServerLive( |
|
| 1811 | + AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet, |
|
| 1812 | + Optional<Action> alsoSendToAllUsersWithThisPermissionOnReplicaSet) throws MailException { |
|
| 1813 | + sendMailToReplicaSetOwner(replicaSet, "NewArchiveServerLiveSubject", "NewArchiveServerLiveBody", |
|
| 1814 | + alsoSendToAllUsersWithThisPermissionOnReplicaSet); |
|
| 1815 | + } |
|
| 1799 | 1816 | |
| 1800 | 1817 | /** |
| 1801 | 1818 | * If a non-{@code null}, non-{@link String#isEmpty() empty} bearer token is provided by the |
| ... | ... | @@ -2140,4 +2157,9 @@ public class LandscapeServiceImpl implements LandscapeService { |
| 2140 | 2157 | final Integer memoryInMegabytes = JvmUtils.getMegabytesFromJvmSize(memoryInMegabytesAsString).orElse(null); |
| 2141 | 2158 | return memoryInMegabytes; |
| 2142 | 2159 | } |
| 2160 | + |
|
| 2161 | + @Override |
|
| 2162 | + public SailingServerFactory getSailingServerFactory() { |
|
| 2163 | + return sailingServerFactoryTracker.getService(); |
|
| 2164 | + } |
|
| 2143 | 2165 | } |
java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/CompareServersResource.java
| ... | ... | @@ -19,9 +19,11 @@ import java.util.UUID; |
| 19 | 19 | import java.util.logging.Logger; |
| 20 | 20 | |
| 21 | 21 | import javax.ws.rs.FormParam; |
| 22 | +import javax.ws.rs.GET; |
|
| 22 | 23 | import javax.ws.rs.POST; |
| 23 | 24 | import javax.ws.rs.Path; |
| 24 | 25 | import javax.ws.rs.Produces; |
| 26 | +import javax.ws.rs.QueryParam; |
|
| 25 | 27 | import javax.ws.rs.core.Context; |
| 26 | 28 | import javax.ws.rs.core.Response; |
| 27 | 29 | import javax.ws.rs.core.Response.Status; |
| ... | ... | @@ -106,6 +108,25 @@ public class CompareServersResource extends AbstractSailingServerResource { |
| 106 | 108 | UriInfo uriInfo; |
| 107 | 109 | |
| 108 | 110 | /** |
| 111 | + * Forwards to the POST method; user authentication is taken from the request's authentication. Therefore, no |
|
| 112 | + * separate authentications for the two servers to compare can be provided. The server receiving this request will |
|
| 113 | + * act as the default for "server1". This is a convenience method for quick comparisons of two |
|
| 114 | + * servers without needing to provide authentication information, assuming that the server receiving this request |
|
| 115 | + * shares its security service with the server specified as "server2" through replication, and that the user |
|
| 116 | + * authenticated for this request has access to both servers. For more complex use cases, use the POST method |
|
| 117 | + * directly. |
|
| 118 | + */ |
|
| 119 | + @GET |
|
| 120 | + @Produces("application/json;charset=UTF-8") |
|
| 121 | + public Response compareServersGet( |
|
| 122 | + @QueryParam(SERVER1_FORM_PARAM) String server1, |
|
| 123 | + @QueryParam(SERVER2_FORM_PARAM) String server2, |
|
| 124 | + @QueryParam(LEADERBOARDGROUP_UUID_FORM_PARAM) Set<String> uuidset) throws MalformedURLException { |
|
| 125 | + return compareServers(server1, server2, uuidset, /* user1 */ null, /* user2 */ null, /* password1 */ null, |
|
| 126 | + /* password2 */ null, /* bearer1 */ null, /* bearer2 */ null); |
|
| 127 | + } |
|
| 128 | + |
|
| 129 | + /** |
|
| 109 | 130 | * @param server1 |
| 110 | 131 | * optional; if not provided, the server receiving this request will act as the default for "server1" |
| 111 | 132 | * @param user1 |
| ... | ... | @@ -204,7 +225,7 @@ public class CompareServersResource extends AbstractSailingServerResource { |
| 204 | 225 | } |
| 205 | 226 | } |
| 206 | 227 | } |
| 207 | - JSONObject json = new JSONObject(); |
|
| 228 | + final JSONObject json = new JSONObject(); |
|
| 208 | 229 | for (Entry<String, Set<Object>> entry : result.entrySet()) { |
| 209 | 230 | json.put(entry.getKey(), entry.getValue()); |
| 210 | 231 | } |
java/com.sap.sailing.server.gateway/webservices/api/v1/compareServers.html
| ... | ... | @@ -33,7 +33,7 @@ |
| 33 | 33 | <td>json</td> |
| 34 | 34 | </tr> |
| 35 | 35 | <tr> |
| 36 | - <td>Mandatory query parameters:</td> |
|
| 36 | + <td>Mandatory form parameters:</td> |
|
| 37 | 37 | <td> |
| 38 | 38 | <div>server2, the hostname for the second server.</div> |
| 39 | 39 | </td> |
| ... | ... | @@ -146,6 +146,48 @@ |
| 146 | 146 | </pre> |
| 147 | 147 | </td> |
| 148 | 148 | </tr> |
| 149 | + <tr> |
|
| 150 | + <td>Webservice Type:</td> |
|
| 151 | + <td>GET</td> |
|
| 152 | + </tr> |
|
| 153 | + <tr> |
|
| 154 | + <td>Output format:</td> |
|
| 155 | + <td>json</td> |
|
| 156 | + </tr> |
|
| 157 | + <tr> |
|
| 158 | + <td>Mandatory query parameters:</td> |
|
| 159 | + <td> |
|
| 160 | + <div>server2, the hostname for the second server.</div> |
|
| 161 | + </td> |
|
| 162 | + </tr> |
|
| 163 | + <tr> |
|
| 164 | + <td>Optional query parameters:</td> |
|
| 165 | + <td> |
|
| 166 | + <div><tt>server1</tt>, the hostname for the first server. Defaults to the target of your request</div> |
|
| 167 | + <div><tt>leaderboardgroupUUID[]</tt>, the leaderboardgroup UUID you want to compare. |
|
| 168 | + Use multiple times if you need to compare more than one leaderboardgroup. Don't specify if you'd like |
|
| 169 | + to compare <em>all</em> leaderboard groups of both servers with each other.</div> |
|
| 170 | + </td> |
|
| 171 | + The GET method can easily be used from a Browser UI. It uses any session/cookie credentials or may be used |
|
| 172 | + with basic authentication. No authentication information can be provided as query parameters with the GET method. |
|
| 173 | + </tr> |
|
| 174 | + <tr> |
|
| 175 | + <td>Request method:</td> |
|
| 176 | + <td>GET</td> |
|
| 177 | + </tr> |
|
| 178 | + <tr> |
|
| 179 | + <td>Example:</td> |
|
| 180 | + <td>curl http://127.0.0.1:8888/sailingserver/api/v1/compareservers?server2=127.0.0.1:8889&leaderboardgroupUUID[]=0334e85e-ebc9-4835-b44d-5db71245b9e9&leaderboardgroupUUID[]=78476ac8-5519-4c3f-9c6d-670468edf4fc" -H "Authorization: Basic YWRtaW46YWRtaW4="<br> |
|
| 181 | + produces output:<pre> |
|
| 182 | + { |
|
| 183 | + 127.0.0.1:8889 [] |
|
| 184 | + 127.0.0.1:8888 [] |
|
| 185 | + } |
|
| 186 | + </pre> |
|
| 187 | + </td> |
|
| 188 | + </tr> |
|
| 189 | + </td> |
|
| 190 | + </tr> |
|
| 149 | 191 | </table> |
| 150 | 192 | <div style="height: 1em;"></div> |
| 151 | 193 | <a href="index.html">Back to Web Service Overview</a> |