Changes in eb769b0: Merge branch 'main' into bug6209

				configuration/buildAndUpdateProduct.sh
			
          @@ -671,7 +671,7 @@ if [[ "$@" == "build" ]] || [[ "$@" == "all" ]]; then

                   PATH=$PATH:$ANDROID_HOME/platform-tools

                   SDK_MANAGER="$ANDROID_HOME/cmdline-tools/8.0/bin/sdkmanager"

                   if [ \! -x "$SDK_MANAGER" ]; then

          -            SDK_MANAGER="$ANDROID_HOME/tools/bin/sdkmanager.bat"

          +            SDK_MANAGER="$ANDROID_HOME/tools/bin/sdkmanager"

                   fi

                   echo "SDK_MANAGER=${SDK_MANAGER}"

                   echo "cmdline-tools:"

				java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/server/StatusServlet.java
			
          @@ -23,6 +23,8 @@ import com.mongodb.connection.ClusterDescription;

           import com.mongodb.connection.ServerDescription;

           import com.sap.sailing.server.interfaces.RacingEventService;

           import com.sap.sse.ServerInfo;

          +import com.sap.sse.common.Duration;

          +import com.sap.sse.common.Util;

           import com.sap.sse.mongodb.MongoDBService;

           import com.sap.sse.replication.ReplicationService;

           import com.sap.sse.replication.ReplicationStatus;

          @@ -66,11 +68,15 @@ public class StatusServlet extends HttpServlet {

                       if (defaultBackgroundTaskThreadPoolExecutor instanceof ThreadPoolExecutor) {

                           final long queueLengthDefaultBackgroundThreadPoolExecutor = ((ThreadPoolExecutor) defaultBackgroundTaskThreadPoolExecutor).getQueue().size();

                           result.put("defaultbackgroundthreadpoolexecutorqueuelength", queueLengthDefaultBackgroundThreadPoolExecutor);

          +                final long nonDelayedQueueLengthDefaultBackgroundThreadPoolExecutor = Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan((ThreadPoolExecutor) defaultBackgroundTaskThreadPoolExecutor, Duration.ONE_SECOND));

          +                result.put("defaultbackgroundthreadpoolexecutorqueuelengthnondelayed", nonDelayedQueueLengthDefaultBackgroundThreadPoolExecutor);

                       }

                       final ScheduledExecutorService defaultForegroundTaskThreadPoolExecutor = ThreadPoolUtil.INSTANCE.getDefaultForegroundTaskThreadPoolExecutor();

                       if (defaultForegroundTaskThreadPoolExecutor instanceof ThreadPoolExecutor) {

                           final long queueLengthDefaultForegroundThreadPoolExecutor = ((ThreadPoolExecutor) defaultForegroundTaskThreadPoolExecutor).getQueue().size();

                           result.put("defaultforegroundthreadpoolexecutorqueuelength", queueLengthDefaultForegroundThreadPoolExecutor);

          +                final long nonDelayedQueueLengthDefaultForegroundThreadPoolExecutor = Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan((ThreadPoolExecutor) defaultForegroundTaskThreadPoolExecutor, Duration.ONE_SECOND));

          +                result.put("defaultforegroundthreadpoolexecutorqueuelengthnondelayed", nonDelayedQueueLengthDefaultForegroundThreadPoolExecutor);

                       }

                       final double systemLoadAverageLastMinute = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();

                       result.put("systemloadaveragelastminute", systemLoadAverageLastMinute);

				java/com.sap.sailing.landscape/resources/stringmessages/SailingLandscape_StringMessages.properties
			
          @@ -15,4 +15,6 @@ FinishedToArchiveReplicaSetIntoBody=Archiving a replica set to {0} has finished.

           FinishedToArchiveReplicaSetSubject=Archiving replica set {0} finished

           FinishedToArchiveReplicaSetBody=Archiving replica set {0} has finished.\nIf you requested so, the original replica set has been removed.\n\nYou are receiving this mail because you have administrative permissions for {0}.\nRemove those permissions at <https://sapsailing.com/gwt/AdminConsole.html#UserManagementPlace:> if you do not like to receive these messages anymore.

           NewArchiveCandidateReadyForSpotChecksAndRotationSubject=The new {0} candidate is ready for spot checks

          -NewArchiveCandidateReadyForSpotChecksAndRotationBody=We''ve run the following checks:\n{3}.\nThe new {0} candidate is ready for spot checks and, if OK, rotation to become the new production {0}.\nRun your spot checks at <https://{1}/gwt/Home.html#EventsPlace:> and compare to <https://{2}/gwt/Home.html#EventsPlace:>.\nStart the rotation to the new production server at <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> after successful checks.

          \ No newline at end of file

          +NewArchiveCandidateReadyForSpotChecksAndRotationBody=We''ve run the following checks:\n{3}.\nThe new {0} candidate is ready for spot checks and, if OK, rotation to become the new production {0}.\nRun your spot checks at <https://{1}/gwt/Home.html#EventsPlace:> and compare to <https://{2}/gwt/Home.html#EventsPlace:>.\nStart the rotation to the new production server at <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> after successful checks.

          +NewArchiveCandidateFailedSubject=Check {1} failed during {0} startup

          +NewArchiveCandidateFailedBody=The check "{1}" failed with message "{2}" while starting up {0}. Please check the landscape and fix manually.

          \ No newline at end of file

				java/com.sap.sailing.landscape/resources/stringmessages/SailingLandscape_StringMessages_de.properties
			
          @@ -15,4 +15,6 @@ FinishedToArchiveReplicaSetIntoBody=Die Archivierung des Anwendungs-Clusters nac

           FinishedToArchiveReplicaSetSubject=Archivierung des Anwendungs-Clusters {0} beendet

           FinishedToArchiveReplicaSetBody=Die Archivierung des Anwendungs-Clusters {0} ist beendet.\nFalls angefragt, wurde das Original Anwendungs-Cluster entfernt.\n\nDiese Nachricht wurde versandt, weil Du über administrative Rechte für {0} verfügst.\nUm das zu ändern, besuche <https://sapsailing.com/gwt/AdminConsole.html#UserManagementPlace:>, um diese Rechte für Dein Benutzerkonto zu entfernen.

           NewArchiveCandidateReadyForSpotChecksAndRotationSubject=Der neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich

          -NewArchiveCandidateReadyForSpotChecksAndRotationBody=Es wurden die folgenden Prüfungen durchgeführt:\n{3}.\nDer neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich\nund, falls OK, Rotation zum neuen Produktiv-Server für {0}.\nStichprobenartiger Vergleich unter https://{1}/gwt/Home.html#EventsPlace: und https://{2}/gwt/Home.html#EventsPlace:.\nRotation nach erfolgreichen Prüfungen unter <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> starten.

          \ No newline at end of file

          +NewArchiveCandidateReadyForSpotChecksAndRotationBody=Es wurden die folgenden Prüfungen durchgeführt:\n{3}.\nDer neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich\nund, falls OK, Rotation zum neuen Produktiv-Server für {0}.\nStichprobenartiger Vergleich unter https://{1}/gwt/Home.html#EventsPlace: und https://{2}/gwt/Home.html#EventsPlace:.\nRotation nach erfolgreichen Prüfungen unter <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> starten.

          +NewArchiveCandidateFailedSubject=Überprüfung {1} beim Start von {0} fehlgeschlagen

          +NewArchiveCandidateFailedBody=Die Überprüfung "{1}" zum Start von {0} ist mit der Meldung "{2}" fehlgeschlagen. Bitte die Landschaft inspizieren und manuell korrigieren.

          \ No newline at end of file

				java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/SailingAnalyticsProcess.java
			
          @@ -10,6 +10,7 @@ import com.sap.sailing.landscape.procedures.SailingProcessConfigurationVariables

           import com.sap.sse.common.Duration;

           import com.sap.sse.landscape.Release;

           import com.sap.sse.landscape.aws.AwsApplicationProcess;

          +import com.sap.sse.util.ThreadPoolUtil;

           public interface SailingAnalyticsProcess<ShardingKey> extends AwsApplicationProcess<ShardingKey, SailingAnalyticsMetrics, SailingAnalyticsProcess<ShardingKey>> {

               static Logger logger = Logger.getLogger(SailingAnalyticsProcess.class.getName());

          @@ -40,7 +41,39 @@ public interface SailingAnalyticsProcess<ShardingKey> extends AwsApplicationProc

               double getLastMinuteSystemLoadAverage(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;

          +    /**

          +     * The count of <em>all</em> tasks in the default background thread pool executor's queue, including those scheduled

          +     * with a delay, such as rate limiter and cache clean-up tasks or periodic fetching of remote server content.

          +     * 

          +     * @see ThreadPoolUtil#getDefaultBackgroundTaskThreadPoolExecutor()

          +     */

               int getDefaultBackgroundThreadPoolExecutorQueueSize(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;

          +    /**

          +     * The count of <em>all</em> tasks in the default foreground thread pool executor's queue, including those scheduled

          +     * with a delay, such as rate limiter and cache clean-up tasks or periodic fetching of remote server content.

          +     * 

          +     * @see ThreadPoolUtil#getDefaultForegroundTaskThreadPoolExecutor()

          +     */

               int getDefaultForegroundThreadPoolExecutorQueueSize(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;

          +

          +    /**

          +     * The count of only those tasks in the default background thread pool executor's queue that have no scheduling

          +     * delay or a delay below {@link Duration#ONE_SECOND one second}. This helps exclude from the count those rate

          +     * limiter and cache clean-up tasks or periodic fetching of remote server content. It is as such as good approximation

          +     * of the "immediate" tasks that will keep the server busy in the immediate future.

          +     * 

          +     * @see ThreadPoolUtil#getDefaultBackgroundTaskThreadPoolExecutor()

          +     */

          +    int getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;

          +

          +    /**

          +     * The count of only those tasks in the default foreground thread pool executor's queue that have no scheduling

          +     * delay or a delay below {@link Duration#ONE_SECOND one second}. This helps exclude from the count those rate

          +     * limiter and cache clean-up tasks or periodic fetching of remote server content. It is as such as good approximation

          +     * of the "immediate" tasks that will keep the server busy in the immediate future.

          +     * 

          +     * @see ThreadPoolUtil#getDefaultForegroundTaskThreadPoolExecutor()

          +     */

          +    int getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;

           }

				java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/ArchiveCandidateMonitoringBackgroundTask.java
			
          @@ -140,6 +140,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {

                   this.executor = executor;

                   this.effectiveBearerToken = effectiveBearerToken;

                   this.checks = Arrays.asList(

          +                new IsAlive(),

                           new IsReady(),

                           new HasLowEnoughSystemLoad(),

                           new HasShortEnoughDefaultBackgroundThreadPoolExecutorQueue(),

          @@ -191,11 +192,28 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {

                   }

               }

          +    private class IsAlive extends AbstractCheck {

          +        private static final long serialVersionUID = -4265303532881568290L;

          +

          +        private IsAlive() {

          +            super("is alive", TIMEOUT_FIRST_CONTACT.get(), DELAY_BETWEEN_CHECKS);

          +        }

          +

          +        @Override

          +        public boolean runCheck() throws Exception {

          +            final boolean result = replicaSet.getMaster().isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT);

          +            if (!result) {

          +                setLastFailureMessage("Candidate at "+replicaSet.getMaster().getHost().getPrivateAddress()+" not alive yet");

          +            }

          +            return result;

          +        }

          +    }

          +

               private class IsReady extends AbstractCheck {

                   private static final long serialVersionUID = -4265303532881568290L;

                   private IsReady() {

          -            super("is ready", TIMEOUT_FIRST_CONTACT.get(), DELAY_BETWEEN_CHECKS);

          +            super("is ready", LONG_TIMEOUT, DELAY_BETWEEN_CHECKS);

                   }

                   @Override

          @@ -237,7 +255,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {

                   @Override

                   public boolean runCheck() throws Exception {

          -            final int defaultBackgroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT);

          +            final int defaultBackgroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Landscape.WAIT_FOR_PROCESS_TIMEOUT);

                       final boolean result = defaultBackgroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE;

                       if (!result) {

                           setLastFailureMessage("Candidate at " + replicaSet.getMaster().getHost().getPrivateAddress()

          @@ -257,7 +275,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {

                   @Override

                   public boolean runCheck() throws Exception {

          -            final int defaultForegroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT);

          +            final int defaultForegroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Landscape.WAIT_FOR_PROCESS_TIMEOUT);

                       final boolean result = defaultForegroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE;

                       if (!result) {

                           setLastFailureMessage("Candidate at "+replicaSet.getMaster().getHost().getPrivateAddress()

				java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/SailingAnalyticsProcessImpl.java
			
          @@ -54,6 +54,8 @@ implements SailingAnalyticsProcess<ShardingKey> {

               private static final String SYSTEM_LOAD_AVERAGE_LAST_MINUTE_NAME = "systemloadaveragelastminute";

               private static final String DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NAME = "defaultbackgroundthreadpoolexecutorqueuelength";

               private static final String DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NAME = "defaultforegroundthreadpoolexecutorqueuelength";

          +    private static final String DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME = "defaultbackgroundthreadpoolexecutorqueuelengthnondelayed";

          +    private static final String DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME = "defaultforegroundthreadpoolexecutorqueuelengthnondelayed";

               private Integer expeditionUdpPort;

               private Integer igtimiRiotPort;

               private Release release;

          @@ -148,6 +150,18 @@ implements SailingAnalyticsProcess<ShardingKey> {

               }

               @Override

          +    public int getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception {

          +        final JSONObject status = getStatus(optionalTimeout);

          +        return ((Number) status.get(DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME)).intValue();

          +    }

          +    

          +    @Override

          +    public int getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception {

          +        final JSONObject status = getStatus(optionalTimeout);

          +        return ((Number) status.get(DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME)).intValue();

          +    }

          +    

          +    @Override

               public Database getDatabaseConfiguration(Region region, Optional<Duration> optionalTimeout,

                       Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {

                   final JSONObject mongoDBConfiguration = (JSONObject) getStatus(optionalTimeout).get(MONGODB_CONFIGURATION_PROPERTY_NAME);

				java/com.sap.sse.security/src/com/sap/sse/security/impl/SecurityServiceImpl.java
			
          @@ -1411,17 +1411,18 @@ implements ReplicableSecurityService, ClearStateTestSupport {

               }

               /**

          -     * Schedule a clean-up task to avoid leaking memory for the TimedLock objects; schedule it in two times the

          -     * locking expiry of {@code timedLock}, but at least one hour, because if no authentication failure occurs

          -     * for that IP/user agent combination, we will entirely remove the {@link TimedLock} from the map,

          -     * effectively resetting that IP to a short default locking duration again; this way, if during the double

          -     * expiration time another failed attempt is registered, we can still grow the locking duration because we have kept

          -     * the {@link TimedLock} object available for a bit longer. Furthermore, for authentication requests, the

          -     * responsible {@link Realm} will let authentication requests get to here only if not locked, so if we were to

          -     * expunge entries immediately as they unlock, the locking duration could never grow.<p>

          +     * Schedule a clean-up task to avoid leaking memory for the {@link TimedLock} objects; schedule it in two times the

          +     * locking expiry of {@code timedLock}, but at least one hour, because if no authentication failure occurs for that

          +     * IP/user agent combination, we will entirely remove the {@link TimedLock} from the map, effectively resetting that

          +     * IP to a short default locking duration again; this way, if during the double expiration time another failed

          +     * attempt is registered, we can still grow the locking duration because we have kept the {@link TimedLock} object

          +     * available for a bit longer. Furthermore, for authentication requests, the responsible {@link Realm} will let

          +     * authentication requests get to here only if not locked, so if we were to expunge entries immediately as they

          +     * unlock, the locking duration could never grow.

          +     * <p>

                * 

          -     * With the minimum of one hour, we ensure that failing requests done at a slower rate still grow the locking

          -     * expiry duration.

          +     * With the minimum of one hour, we ensure that failing requests done at a slower rate still grow the locking expiry

          +     * duration.

                */

               private void scheduleCleanUpTask(final String clientIPOrNull,

                       final TimedLock timedLock,

				java/com.sap.sse.test/src/com/sap/sse/test/ThreadPoolNonDelayedTasksCountTest.java
			
          @@ -0,0 +1,70 @@

          +package com.sap.sse.test;

          +

          +import static org.junit.jupiter.api.Assertions.assertEquals;

          +

          +import java.util.concurrent.ScheduledFuture;

          +import java.util.concurrent.ScheduledThreadPoolExecutor;

          +import java.util.concurrent.TimeUnit;

          +

          +import org.junit.jupiter.api.AfterEach;

          +import org.junit.jupiter.api.BeforeEach;

          +import org.junit.jupiter.api.Test;

          +

          +import com.sap.sse.common.Duration;

          +import com.sap.sse.common.Util;

          +import com.sap.sse.util.ThreadPoolUtil;

          +

          +/**

          + * Tests the

          + * {@link ThreadPoolUtil#getTasksDelayedByLessThan(java.util.concurrent.ScheduledExecutorService, com.sap.sse.common.Duration)}

          + * method.

          + * 

          + * @author Axel Uhl (d043530)

          + *

          + */

          +public class ThreadPoolNonDelayedTasksCountTest {

          +    private ScheduledThreadPoolExecutor defaultBackgroundThreadPoolExecutor;

          +    private int corePoolSize;

          +    

          +    @BeforeEach

          +    public void setUp() {

          +        defaultBackgroundThreadPoolExecutor = (ScheduledThreadPoolExecutor) ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor();

          +        corePoolSize = defaultBackgroundThreadPoolExecutor.getCorePoolSize();

          +    }

          +    

          +    @AfterEach

          +    public void tearDown() {

          +        for (final Runnable task : defaultBackgroundThreadPoolExecutor.getQueue()) {

          +            final ScheduledFuture<?> scheduledFuture = (ScheduledFuture<?>) task;

          +            scheduledFuture.cancel(/* mayInterruptIfRunning */ true);

          +        }

          +    }

          +    

          +    @Test

          +    public void testAddingDelayedAndUndelayedThenCounting() throws InterruptedException {

          +        final int IMMEDIATE_TASK_COUNT = 10000;

          +        final int DELAYED_TASK_COUNT = 100;

          +        final Duration DELAY = Duration.ONE_HOUR.times(2);

          +        for (int i=0; i<IMMEDIATE_TASK_COUNT; i++) {

          +            defaultBackgroundThreadPoolExecutor.submit(()->{

          +                try {

          +                    Thread.sleep(IMMEDIATE_TASK_COUNT);

          +                } catch (InterruptedException e) {

          +                    e.printStackTrace();

          +                }

          +            });

          +        }

          +        for (int i=0; i<DELAYED_TASK_COUNT; i++) {

          +            defaultBackgroundThreadPoolExecutor.schedule(()->{

          +                try {

          +                    Thread.sleep(IMMEDIATE_TASK_COUNT);

          +                } catch (InterruptedException e) {

          +                    e.printStackTrace();

          +                }

          +            }, DELAY.asMillis(), TimeUnit.MILLISECONDS);

          +        }

          +        Thread.sleep(100); // wait for non-delayed tasks to get scheduled

          +        assertEquals(IMMEDIATE_TASK_COUNT-corePoolSize, Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan(defaultBackgroundThreadPoolExecutor, DELAY.divide(2))));

          +        assertEquals(IMMEDIATE_TASK_COUNT+DELAYED_TASK_COUNT-corePoolSize, Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan(defaultBackgroundThreadPoolExecutor, DELAY.times(2))));

          +    }

          +}

				java/com.sap.sse/src/com/sap/sse/util/ThreadPoolUtil.java
			
          @@ -5,9 +5,12 @@ import java.util.concurrent.Callable;

           import java.util.concurrent.ExecutorService;

           import java.util.concurrent.Future;

           import java.util.concurrent.ScheduledExecutorService;

          +import java.util.concurrent.ScheduledFuture;

           import java.util.concurrent.ScheduledThreadPoolExecutor;

          +import java.util.concurrent.ThreadPoolExecutor;

           import java.util.logging.Level;

          +import com.sap.sse.common.Duration;

           import com.sap.sse.util.impl.ThreadPoolUtilImpl;

           public interface ThreadPoolUtil {

          @@ -118,4 +121,11 @@ public interface ThreadPoolUtil {

               Runnable associateWithSubjectIfAny(Runnable runnable);

               <T> Callable<T> associateWithSubjectIfAny(Callable<T> callable);

          +    

          +    /**

          +     * In the {@code executor}'s queue filters tasks for those with a delay less than {@code delayLessThan} and

          +     * returns the corresponding tasks. This can be used, e.g., to judge an executor's immediate workload or

          +     * give an estimate of the future workload mapped over time.

          +     */

          +    Iterable<ScheduledFuture<?>> getTasksDelayedByLessThan(ThreadPoolExecutor executor, Duration delayLessThan);

           }

				java/com.sap.sse/src/com/sap/sse/util/impl/ThreadPoolUtilImpl.java
			
          @@ -8,6 +8,9 @@ import java.util.concurrent.ExecutionException;

           import java.util.concurrent.ExecutorService;

           import java.util.concurrent.Future;

           import java.util.concurrent.ScheduledExecutorService;

          +import java.util.concurrent.ScheduledFuture;

          +import java.util.concurrent.ThreadPoolExecutor;

          +import java.util.concurrent.TimeUnit;

           import java.util.logging.Level;

           import java.util.logging.Logger;

          @@ -15,6 +18,7 @@ import org.apache.shiro.SecurityUtils;

           import org.apache.shiro.UnavailableSecurityManagerException;

           import org.apache.shiro.subject.Subject;

          +import com.sap.sse.common.Duration;

           import com.sap.sse.common.Util;

           import com.sap.sse.util.ThreadPoolUtil;

          @@ -130,4 +134,11 @@ public class ThreadPoolUtilImpl implements ThreadPoolUtil {

               public <T> Callable<T> associateWithSubjectIfAny(Callable<T> callable) {

                   return getSubjectOrNull().map(subject->subject.associateWith(callable)).orElse(callable);

               }

          +

          +    @Override

          +    public Iterable<ScheduledFuture<?>> getTasksDelayedByLessThan(ThreadPoolExecutor executor,

          +            Duration delayLessThan) {

          +        return Util.map(Util.filter(executor.getQueue(), task->((ScheduledFuture<?>) task).getDelay(TimeUnit.MILLISECONDS) < delayLessThan.asMillis()),

          +                task->(ScheduledFuture<?>) task);

          +    }

           }

...	...	@@ -671,7 +671,7 @@ if [[ "$@" == "build" ]] \|\| [[ "$@" == "all" ]]; then
671	671	PATH=$PATH:$ANDROID_HOME/platform-tools
672	672	SDK_MANAGER="$ANDROID_HOME/cmdline-tools/8.0/bin/sdkmanager"
673	673	if [ \! -x "$SDK_MANAGER" ]; then
674		- SDK_MANAGER="$ANDROID_HOME/tools/bin/sdkmanager.bat"
	674	+ SDK_MANAGER="$ANDROID_HOME/tools/bin/sdkmanager"
675	675	fi
676	676	echo "SDK_MANAGER=${SDK_MANAGER}"
677	677	echo "cmdline-tools:"

...	...	@@ -140,6 +140,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {
140	140	this.executor = executor;
141	141	this.effectiveBearerToken = effectiveBearerToken;
142	142	this.checks = Arrays.asList(
	143	+ new IsAlive(),
143	144	new IsReady(),
144	145	new HasLowEnoughSystemLoad(),
145	146	new HasShortEnoughDefaultBackgroundThreadPoolExecutorQueue(),
...	...	@@ -191,11 +192,28 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {
191	192	}
192	193	}
193	194
	195	+ private class IsAlive extends AbstractCheck {
	196	+ private static final long serialVersionUID = -4265303532881568290L;
	197	+
	198	+ private IsAlive() {
	199	+ super("is alive", TIMEOUT_FIRST_CONTACT.get(), DELAY_BETWEEN_CHECKS);
	200	+ }
	201	+
	202	+ @Override
	203	+ public boolean runCheck() throws Exception {
	204	+ final boolean result = replicaSet.getMaster().isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT);
	205	+ if (!result) {
	206	+ setLastFailureMessage("Candidate at "+replicaSet.getMaster().getHost().getPrivateAddress()+" not alive yet");
	207	+ }
	208	+ return result;
	209	+ }
	210	+ }
	211	+
194	212	private class IsReady extends AbstractCheck {
195	213	private static final long serialVersionUID = -4265303532881568290L;
196	214
197	215	private IsReady() {
198		- super("is ready", TIMEOUT_FIRST_CONTACT.get(), DELAY_BETWEEN_CHECKS);
	216	+ super("is ready", LONG_TIMEOUT, DELAY_BETWEEN_CHECKS);
199	217	}
200	218
201	219	@Override
...	...	@@ -237,7 +255,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {
237	255
238	256	@Override
239	257	public boolean runCheck() throws Exception {
240		- final int defaultBackgroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT);
	258	+ final int defaultBackgroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Landscape.WAIT_FOR_PROCESS_TIMEOUT);
241	259	final boolean result = defaultBackgroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE;
242	260	if (!result) {
243	261	setLastFailureMessage("Candidate at " + replicaSet.getMaster().getHost().getPrivateAddress()
...	...	@@ -257,7 +275,7 @@ public class ArchiveCandidateMonitoringBackgroundTask implements Runnable {
257	275
258	276	@Override
259	277	public boolean runCheck() throws Exception {
260		- final int defaultForegroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSize(Landscape.WAIT_FOR_PROCESS_TIMEOUT);
	278	+ final int defaultForegroundThreadPoolExecutorQueueSize = replicaSet.getMaster().getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Landscape.WAIT_FOR_PROCESS_TIMEOUT);
261	279	final boolean result = defaultForegroundThreadPoolExecutorQueueSize < MAXIMUM_THREAD_POOL_QUEUE_SIZE;
262	280	if (!result) {
263	281	setLastFailureMessage("Candidate at "+replicaSet.getMaster().getHost().getPrivateAddress()

...	...	@@ -1411,17 +1411,18 @@ implements ReplicableSecurityService, ClearStateTestSupport {
1411	1411	}
1412	1412
1413	1413	/**
1414		- * Schedule a clean-up task to avoid leaking memory for the TimedLock objects; schedule it in two times the
1415		- * locking expiry of {@code timedLock}, but at least one hour, because if no authentication failure occurs
1416		- * for that IP/user agent combination, we will entirely remove the {@link TimedLock} from the map,
1417		- * effectively resetting that IP to a short default locking duration again; this way, if during the double
1418		- * expiration time another failed attempt is registered, we can still grow the locking duration because we have kept
1419		- * the {@link TimedLock} object available for a bit longer. Furthermore, for authentication requests, the
1420		- * responsible {@link Realm} will let authentication requests get to here only if not locked, so if we were to
1421		- * expunge entries immediately as they unlock, the locking duration could never grow.<p>
	1414	+ * Schedule a clean-up task to avoid leaking memory for the {@link TimedLock} objects; schedule it in two times the
	1415	+ * locking expiry of {@code timedLock}, but at least one hour, because if no authentication failure occurs for that
	1416	+ * IP/user agent combination, we will entirely remove the {@link TimedLock} from the map, effectively resetting that
	1417	+ * IP to a short default locking duration again; this way, if during the double expiration time another failed
	1418	+ * attempt is registered, we can still grow the locking duration because we have kept the {@link TimedLock} object
	1419	+ * available for a bit longer. Furthermore, for authentication requests, the responsible {@link Realm} will let
	1420	+ * authentication requests get to here only if not locked, so if we were to expunge entries immediately as they
	1421	+ * unlock, the locking duration could never grow.
	1422	+ * <p>
1422	1423	*
1423		- * With the minimum of one hour, we ensure that failing requests done at a slower rate still grow the locking
1424		- * expiry duration.
	1424	+ * With the minimum of one hour, we ensure that failing requests done at a slower rate still grow the locking expiry
	1425	+ * duration.
1425	1426	*/
1426	1427	private void scheduleCleanUpTask(final String clientIPOrNull,
1427	1428	final TimedLock timedLock,

...	...	@@ -23,6 +23,8 @@ import com.mongodb.connection.ClusterDescription;
23	23	import com.mongodb.connection.ServerDescription;
24	24	import com.sap.sailing.server.interfaces.RacingEventService;
25	25	import com.sap.sse.ServerInfo;
	26	+import com.sap.sse.common.Duration;
	27	+import com.sap.sse.common.Util;
26	28	import com.sap.sse.mongodb.MongoDBService;
27	29	import com.sap.sse.replication.ReplicationService;
28	30	import com.sap.sse.replication.ReplicationStatus;
...	...	@@ -66,11 +68,15 @@ public class StatusServlet extends HttpServlet {
66	68	if (defaultBackgroundTaskThreadPoolExecutor instanceof ThreadPoolExecutor) {
67	69	final long queueLengthDefaultBackgroundThreadPoolExecutor = ((ThreadPoolExecutor) defaultBackgroundTaskThreadPoolExecutor).getQueue().size();
68	70	result.put("defaultbackgroundthreadpoolexecutorqueuelength", queueLengthDefaultBackgroundThreadPoolExecutor);
	71	+ final long nonDelayedQueueLengthDefaultBackgroundThreadPoolExecutor = Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan((ThreadPoolExecutor) defaultBackgroundTaskThreadPoolExecutor, Duration.ONE_SECOND));
	72	+ result.put("defaultbackgroundthreadpoolexecutorqueuelengthnondelayed", nonDelayedQueueLengthDefaultBackgroundThreadPoolExecutor);
69	73	}
70	74	final ScheduledExecutorService defaultForegroundTaskThreadPoolExecutor = ThreadPoolUtil.INSTANCE.getDefaultForegroundTaskThreadPoolExecutor();
71	75	if (defaultForegroundTaskThreadPoolExecutor instanceof ThreadPoolExecutor) {
72	76	final long queueLengthDefaultForegroundThreadPoolExecutor = ((ThreadPoolExecutor) defaultForegroundTaskThreadPoolExecutor).getQueue().size();
73	77	result.put("defaultforegroundthreadpoolexecutorqueuelength", queueLengthDefaultForegroundThreadPoolExecutor);
	78	+ final long nonDelayedQueueLengthDefaultForegroundThreadPoolExecutor = Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan((ThreadPoolExecutor) defaultForegroundTaskThreadPoolExecutor, Duration.ONE_SECOND));
	79	+ result.put("defaultforegroundthreadpoolexecutorqueuelengthnondelayed", nonDelayedQueueLengthDefaultForegroundThreadPoolExecutor);
74	80	}
75	81	final double systemLoadAverageLastMinute = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
76	82	result.put("systemloadaveragelastminute", systemLoadAverageLastMinute);

...	...	@@ -15,4 +15,6 @@ FinishedToArchiveReplicaSetIntoBody=Archiving a replica set to {0} has finished.
15	15	FinishedToArchiveReplicaSetSubject=Archiving replica set {0} finished
16	16	FinishedToArchiveReplicaSetBody=Archiving replica set {0} has finished.\nIf you requested so, the original replica set has been removed.\n\nYou are receiving this mail because you have administrative permissions for {0}.\nRemove those permissions at <https://sapsailing.com/gwt/AdminConsole.html#UserManagementPlace:> if you do not like to receive these messages anymore.
17	17	NewArchiveCandidateReadyForSpotChecksAndRotationSubject=The new {0} candidate is ready for spot checks
18		-NewArchiveCandidateReadyForSpotChecksAndRotationBody=We''ve run the following checks:\n{3}.\nThe new {0} candidate is ready for spot checks and, if OK, rotation to become the new production {0}.\nRun your spot checks at <https://{1}/gwt/Home.html#EventsPlace:> and compare to <https://{2}/gwt/Home.html#EventsPlace:>.\nStart the rotation to the new production server at <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> after successful checks.
...	...	\ No newline at end of file
	0	+NewArchiveCandidateReadyForSpotChecksAndRotationBody=We''ve run the following checks:\n{3}.\nThe new {0} candidate is ready for spot checks and, if OK, rotation to become the new production {0}.\nRun your spot checks at <https://{1}/gwt/Home.html#EventsPlace:> and compare to <https://{2}/gwt/Home.html#EventsPlace:>.\nStart the rotation to the new production server at <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> after successful checks.
	1	+NewArchiveCandidateFailedSubject=Check {1} failed during {0} startup
	2	+NewArchiveCandidateFailedBody=The check "{1}" failed with message "{2}" while starting up {0}. Please check the landscape and fix manually.
...	...	\ No newline at end of file

...	...	@@ -15,4 +15,6 @@ FinishedToArchiveReplicaSetIntoBody=Die Archivierung des Anwendungs-Clusters nac
15	15	FinishedToArchiveReplicaSetSubject=Archivierung des Anwendungs-Clusters {0} beendet
16	16	FinishedToArchiveReplicaSetBody=Die Archivierung des Anwendungs-Clusters {0} ist beendet.\nFalls angefragt, wurde das Original Anwendungs-Cluster entfernt.\n\nDiese Nachricht wurde versandt, weil Du über administrative Rechte für {0} verfügst.\nUm das zu ändern, besuche <https://sapsailing.com/gwt/AdminConsole.html#UserManagementPlace:>, um diese Rechte für Dein Benutzerkonto zu entfernen.
17	17	NewArchiveCandidateReadyForSpotChecksAndRotationSubject=Der neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich
18		-NewArchiveCandidateReadyForSpotChecksAndRotationBody=Es wurden die folgenden Prüfungen durchgeführt:\n{3}.\nDer neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich\nund, falls OK, Rotation zum neuen Produktiv-Server für {0}.\nStichprobenartiger Vergleich unter https://{1}/gwt/Home.html#EventsPlace: und https://{2}/gwt/Home.html#EventsPlace:.\nRotation nach erfolgreichen Prüfungen unter <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> starten.
...	...	\ No newline at end of file
	0	+NewArchiveCandidateReadyForSpotChecksAndRotationBody=Es wurden die folgenden Prüfungen durchgeführt:\n{3}.\nDer neue {0} Kandidat ist bereit für einen stichprobenartigen Vergleich\nund, falls OK, Rotation zum neuen Produktiv-Server für {0}.\nStichprobenartiger Vergleich unter https://{1}/gwt/Home.html#EventsPlace: und https://{2}/gwt/Home.html#EventsPlace:.\nRotation nach erfolgreichen Prüfungen unter <{4}/gwt/AdminConsole.html#LandscapeManagementPlace:> starten.
	1	+NewArchiveCandidateFailedSubject=Überprüfung {1} beim Start von {0} fehlgeschlagen
	2	+NewArchiveCandidateFailedBody=Die Überprüfung "{1}" zum Start von {0} ist mit der Meldung "{2}" fehlgeschlagen. Bitte die Landschaft inspizieren und manuell korrigieren.
...	...	\ No newline at end of file

...	...	@@ -10,6 +10,7 @@ import com.sap.sailing.landscape.procedures.SailingProcessConfigurationVariables
10	10	import com.sap.sse.common.Duration;
11	11	import com.sap.sse.landscape.Release;
12	12	import com.sap.sse.landscape.aws.AwsApplicationProcess;
	13	+import com.sap.sse.util.ThreadPoolUtil;
13	14
14	15	public interface SailingAnalyticsProcess<ShardingKey> extends AwsApplicationProcess<ShardingKey, SailingAnalyticsMetrics, SailingAnalyticsProcess<ShardingKey>> {
15	16	static Logger logger = Logger.getLogger(SailingAnalyticsProcess.class.getName());
...	...	@@ -40,7 +41,39 @@ public interface SailingAnalyticsProcess<ShardingKey> extends AwsApplicationProc
40	41
41	42	double getLastMinuteSystemLoadAverage(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;
42	43
	44	+ /**
	45	+ * The count of <em>all</em> tasks in the default background thread pool executor's queue, including those scheduled
	46	+ * with a delay, such as rate limiter and cache clean-up tasks or periodic fetching of remote server content.
	47	+ *
	48	+ * @see ThreadPoolUtil#getDefaultBackgroundTaskThreadPoolExecutor()
	49	+ */
43	50	int getDefaultBackgroundThreadPoolExecutorQueueSize(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;
44	51
	52	+ /**
	53	+ * The count of <em>all</em> tasks in the default foreground thread pool executor's queue, including those scheduled
	54	+ * with a delay, such as rate limiter and cache clean-up tasks or periodic fetching of remote server content.
	55	+ *
	56	+ * @see ThreadPoolUtil#getDefaultForegroundTaskThreadPoolExecutor()
	57	+ */
45	58	int getDefaultForegroundThreadPoolExecutorQueueSize(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;
	59	+
	60	+ /**
	61	+ * The count of only those tasks in the default background thread pool executor's queue that have no scheduling
	62	+ * delay or a delay below {@link Duration#ONE_SECOND one second}. This helps exclude from the count those rate
	63	+ * limiter and cache clean-up tasks or periodic fetching of remote server content. It is as such as good approximation
	64	+ * of the "immediate" tasks that will keep the server busy in the immediate future.
	65	+ *
	66	+ * @see ThreadPoolUtil#getDefaultBackgroundTaskThreadPoolExecutor()
	67	+ */
	68	+ int getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;
	69	+
	70	+ /**
	71	+ * The count of only those tasks in the default foreground thread pool executor's queue that have no scheduling
	72	+ * delay or a delay below {@link Duration#ONE_SECOND one second}. This helps exclude from the count those rate
	73	+ * limiter and cache clean-up tasks or periodic fetching of remote server content. It is as such as good approximation
	74	+ * of the "immediate" tasks that will keep the server busy in the immediate future.
	75	+ *
	76	+ * @see ThreadPoolUtil#getDefaultForegroundTaskThreadPoolExecutor()
	77	+ */
	78	+ int getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception;
46	79	}

...	...	@@ -54,6 +54,8 @@ implements SailingAnalyticsProcess<ShardingKey> {
54	54	private static final String SYSTEM_LOAD_AVERAGE_LAST_MINUTE_NAME = "systemloadaveragelastminute";
55	55	private static final String DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NAME = "defaultbackgroundthreadpoolexecutorqueuelength";
56	56	private static final String DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NAME = "defaultforegroundthreadpoolexecutorqueuelength";
	57	+ private static final String DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME = "defaultbackgroundthreadpoolexecutorqueuelengthnondelayed";
	58	+ private static final String DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME = "defaultforegroundthreadpoolexecutorqueuelengthnondelayed";
57	59	private Integer expeditionUdpPort;
58	60	private Integer igtimiRiotPort;
59	61	private Release release;
...	...	@@ -148,6 +150,18 @@ implements SailingAnalyticsProcess<ShardingKey> {
148	150	}
149	151
150	152	@Override
	153	+ public int getDefaultBackgroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception {
	154	+ final JSONObject status = getStatus(optionalTimeout);
	155	+ return ((Number) status.get(DEFAULT_BACKGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME)).intValue();
	156	+ }
	157	+
	158	+ @Override
	159	+ public int getDefaultForegroundThreadPoolExecutorQueueSizeNondelayed(Optional<Duration> optionalTimeout) throws TimeoutException, Exception {
	160	+ final JSONObject status = getStatus(optionalTimeout);
	161	+ return ((Number) status.get(DEFAULT_FOREGROUND_THREAD_POOL_EXECUTOR_QUEUE_LENGTH_NONDELAYED_NAME)).intValue();
	162	+ }
	163	+
	164	+ @Override
151	165	public Database getDatabaseConfiguration(Region region, Optional<Duration> optionalTimeout,
152	166	Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception {
153	167	final JSONObject mongoDBConfiguration = (JSONObject) getStatus(optionalTimeout).get(MONGODB_CONFIGURATION_PROPERTY_NAME);

...	...	@@ -0,0 +1,70 @@
	1	+package com.sap.sse.test;
	2	+
	3	+import static org.junit.jupiter.api.Assertions.assertEquals;
	4	+
	5	+import java.util.concurrent.ScheduledFuture;
	6	+import java.util.concurrent.ScheduledThreadPoolExecutor;
	7	+import java.util.concurrent.TimeUnit;
	8	+
	9	+import org.junit.jupiter.api.AfterEach;
	10	+import org.junit.jupiter.api.BeforeEach;
	11	+import org.junit.jupiter.api.Test;
	12	+
	13	+import com.sap.sse.common.Duration;
	14	+import com.sap.sse.common.Util;
	15	+import com.sap.sse.util.ThreadPoolUtil;
	16	+
	17	+/**
	18	+ * Tests the
	19	+ * {@link ThreadPoolUtil#getTasksDelayedByLessThan(java.util.concurrent.ScheduledExecutorService, com.sap.sse.common.Duration)}
	20	+ * method.
	21	+ *
	22	+ * @author Axel Uhl (d043530)
	23	+ *
	24	+ */
	25	+public class ThreadPoolNonDelayedTasksCountTest {
	26	+ private ScheduledThreadPoolExecutor defaultBackgroundThreadPoolExecutor;
	27	+ private int corePoolSize;
	28	+
	29	+ @BeforeEach
	30	+ public void setUp() {
	31	+ defaultBackgroundThreadPoolExecutor = (ScheduledThreadPoolExecutor) ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor();
	32	+ corePoolSize = defaultBackgroundThreadPoolExecutor.getCorePoolSize();
	33	+ }
	34	+
	35	+ @AfterEach
	36	+ public void tearDown() {
	37	+ for (final Runnable task : defaultBackgroundThreadPoolExecutor.getQueue()) {
	38	+ final ScheduledFuture<?> scheduledFuture = (ScheduledFuture<?>) task;
	39	+ scheduledFuture.cancel(/* mayInterruptIfRunning */ true);
	40	+ }
	41	+ }
	42	+
	43	+ @Test
	44	+ public void testAddingDelayedAndUndelayedThenCounting() throws InterruptedException {
	45	+ final int IMMEDIATE_TASK_COUNT = 10000;
	46	+ final int DELAYED_TASK_COUNT = 100;
	47	+ final Duration DELAY = Duration.ONE_HOUR.times(2);
	48	+ for (int i=0; i<IMMEDIATE_TASK_COUNT; i++) {
	49	+ defaultBackgroundThreadPoolExecutor.submit(()->{
	50	+ try {
	51	+ Thread.sleep(IMMEDIATE_TASK_COUNT);
	52	+ } catch (InterruptedException e) {
	53	+ e.printStackTrace();
	54	+ }
	55	+ });
	56	+ }
	57	+ for (int i=0; i<DELAYED_TASK_COUNT; i++) {
	58	+ defaultBackgroundThreadPoolExecutor.schedule(()->{
	59	+ try {
	60	+ Thread.sleep(IMMEDIATE_TASK_COUNT);
	61	+ } catch (InterruptedException e) {
	62	+ e.printStackTrace();
	63	+ }
	64	+ }, DELAY.asMillis(), TimeUnit.MILLISECONDS);
	65	+ }
	66	+ Thread.sleep(100); // wait for non-delayed tasks to get scheduled
	67	+ assertEquals(IMMEDIATE_TASK_COUNT-corePoolSize, Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan(defaultBackgroundThreadPoolExecutor, DELAY.divide(2))));
	68	+ assertEquals(IMMEDIATE_TASK_COUNT+DELAYED_TASK_COUNT-corePoolSize, Util.size(ThreadPoolUtil.INSTANCE.getTasksDelayedByLessThan(defaultBackgroundThreadPoolExecutor, DELAY.times(2))));
	69	+ }
	70	+}

...	...	@@ -5,9 +5,12 @@ import java.util.concurrent.Callable;
5	5	import java.util.concurrent.ExecutorService;
6	6	import java.util.concurrent.Future;
7	7	import java.util.concurrent.ScheduledExecutorService;
	8	+import java.util.concurrent.ScheduledFuture;
8	9	import java.util.concurrent.ScheduledThreadPoolExecutor;
	10	+import java.util.concurrent.ThreadPoolExecutor;
9	11	import java.util.logging.Level;
10	12
	13	+import com.sap.sse.common.Duration;
11	14	import com.sap.sse.util.impl.ThreadPoolUtilImpl;
12	15
13	16	public interface ThreadPoolUtil {
...	...	@@ -118,4 +121,11 @@ public interface ThreadPoolUtil {
118	121	Runnable associateWithSubjectIfAny(Runnable runnable);
119	122
120	123	<T> Callable<T> associateWithSubjectIfAny(Callable<T> callable);
	124	+
	125	+ /**
	126	+ * In the {@code executor}'s queue filters tasks for those with a delay less than {@code delayLessThan} and
	127	+ * returns the corresponding tasks. This can be used, e.g., to judge an executor's immediate workload or
	128	+ * give an estimate of the future workload mapped over time.
	129	+ */
	130	+ Iterable<ScheduledFuture<?>> getTasksDelayedByLessThan(ThreadPoolExecutor executor, Duration delayLessThan);
121	131	}

...	...	@@ -8,6 +8,9 @@ import java.util.concurrent.ExecutionException;
8	8	import java.util.concurrent.ExecutorService;
9	9	import java.util.concurrent.Future;
10	10	import java.util.concurrent.ScheduledExecutorService;
	11	+import java.util.concurrent.ScheduledFuture;
	12	+import java.util.concurrent.ThreadPoolExecutor;
	13	+import java.util.concurrent.TimeUnit;
11	14	import java.util.logging.Level;
12	15	import java.util.logging.Logger;
13	16
...	...	@@ -15,6 +18,7 @@ import org.apache.shiro.SecurityUtils;
15	18	import org.apache.shiro.UnavailableSecurityManagerException;
16	19	import org.apache.shiro.subject.Subject;
17	20
	21	+import com.sap.sse.common.Duration;
18	22	import com.sap.sse.common.Util;
19	23	import com.sap.sse.util.ThreadPoolUtil;
20	24
...	...	@@ -130,4 +134,11 @@ public class ThreadPoolUtilImpl implements ThreadPoolUtil {
130	134	public <T> Callable<T> associateWithSubjectIfAny(Callable<T> callable) {
131	135	return getSubjectOrNull().map(subject->subject.associateWith(callable)).orElse(callable);
132	136	}
	137	+
	138	+ @Override
	139	+ public Iterable<ScheduledFuture<?>> getTasksDelayedByLessThan(ThreadPoolExecutor executor,
	140	+ Duration delayLessThan) {
	141	+ return Util.map(Util.filter(executor.getQueue(), task->((ScheduledFuture<?>) task).getDelay(TimeUnit.MILLISECONDS) < delayLessThan.asMillis()),
	142	+ task->(ScheduledFuture<?>) task);
	143	+ }
133	144	}