java/com.sap.sse.landscape.test/src/com/sap/sse/landscape/impl/TestGithubReleaseRepository.java
... ...
@@ -5,12 +5,19 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
5 5
import static org.junit.jupiter.api.Assertions.assertNotNull;
6 6
import static org.junit.jupiter.api.Assertions.assertNull;
7 7
8
+import java.util.HashMap;
9
+import java.util.Map;
10
+import java.util.concurrent.ExecutionException;
11
+import java.util.concurrent.Future;
12
+import java.util.concurrent.ScheduledExecutorService;
13
+
8 14
import org.junit.jupiter.api.BeforeAll;
9 15
import org.junit.jupiter.api.Disabled;
10 16
import org.junit.jupiter.api.Test;
11 17
12 18
import com.sap.sse.common.Util;
13 19
import com.sap.sse.landscape.Release;
20
+import com.sap.sse.util.ThreadPoolUtil;
14 21
15 22
public class TestGithubReleaseRepository {
16 23
private static final String DOCKER_25 = "docker-25";
... ...
@@ -64,4 +71,19 @@ public class TestGithubReleaseRepository {
64 71
public void testOldDocker17ReleaseExists() {
65 72
assertFalse(Util.isEmpty(Util.filter(repository, release->release.getName().equals("docker-17-202404262046"))));
66 73
}
74
+
75
+ @Test
76
+ public void testConcurrentAccess() throws InterruptedException, ExecutionException {
77
+ final ScheduledExecutorService threadPool = ThreadPoolUtil.INSTANCE.createForegroundTaskThreadPoolExecutor(10, getClass().getName()+":testConcurrentAccess()");
78
+ final Map<String, Future<Release>> futures = new HashMap<>();
79
+ final String[] prefixes = new String[] { "main", "docker-25", "docker-24", "docker-21", "docker-17" };
80
+ for (final String prefix : prefixes) {
81
+ futures.put(prefix, threadPool.submit(()->repository.getLatestRelease(prefix)));
82
+ }
83
+ for (final String prefix : prefixes) {
84
+ assertNotNull(futures.get(prefix).get());
85
+ assertEquals(prefix, futures.get(prefix).get().getBaseName());
86
+ }
87
+ threadPool.shutdown();
88
+ }
67 89
}
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/GithubReleasesRepository.java
... ...
@@ -9,7 +9,8 @@ import java.net.URLConnection;
9 9
import java.text.SimpleDateFormat;
10 10
import java.util.Iterator;
11 11
import java.util.NoSuchElementException;
12
-import java.util.TreeMap;
12
+import java.util.concurrent.ConcurrentNavigableMap;
13
+import java.util.concurrent.ConcurrentSkipListMap;
13 14
import java.util.logging.Logger;
14 15
import java.util.regex.Matcher;
15 16
import java.util.regex.Pattern;
... ...
@@ -27,13 +28,35 @@ import com.sap.sse.landscape.ReleaseRepository;
27 28
import com.sap.sse.util.HttpUrlConnectionHelper;
28 29
29 30
/**
30
- * Assumes a public GitHub repository where releases can be freely downloaded from
31
- * <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>. The GitHub
32
- * {@code /releases} end point delivers the releases in descending chronological order, so
33
- * newest releases first. With this, we can cache old results and try to get along with the
34
- * harsh rate limit of only 60 requests per hour when used without authentication.<p>
31
+ * Can enumerate the {@link Release}s published by a GitHub repository and search releases whose name starts with a
32
+ * specific prefix. This assumes a public GitHub repository where releases can be freely downloaded from
33
+ * <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>. The {@code api.github.com}'s
34
+ * {@code /releases} end point delivers the releases in descending chronological order, so newest releases first. With
35
+ * this, we can cache old results and try to get along with the harsh rate limit of only 60 requests per hour when used
36
+ * without authentication.
37
+ * <p>
35 38
*
36
- * TODO Concurrency Control! What, if multiple requests or iterations are run on this repository object concurrently?<p>
39
+ * Due to the harsh rate limits we restrict loading even of the first page to once every two minutes; multiple requests
40
+ * within this duration will be answered from the cache. With this, a single instance of this class will typically
41
+ * request <em>all</em> releases only once, cache all these releases, and then look for newer releases at most every two
42
+ * minutes, thereby staying well within limits.
43
+ * <p>
44
+ *
45
+ * Enumerating the releases works through the inner class {@link ReleaseIterator}. If the last loading request for the
46
+ * first page happened more than those two minutes ago, another such request will be made and the new, yet uncached
47
+ * releases obtained from it will be added to the cache. Then, enumeration starts on the cache, delivering the newest
48
+ * release first. When the oldest cached element has been delivered through the iterator, the next action depends on
49
+ * whether or not the cache {@link #cacheContainsOldestRelease contains the oldest release} already. If so, no older
50
+ * release can exist, and iteration ends. Otherwise, more requests for further paginated release documents are sent
51
+ * until no more pages are found or releases older than the so far oldest release from the cache are found and added to
52
+ * the cache. Iteration then continues on the cache again.
53
+ * <p>
54
+ *
55
+ * The class is thread-safe in that it allows multiple threads to obtain iterators on a single instance of this class.
56
+ * The loading and caching of releases pages from GitHub, the invocation of the {@link #iterator()} method and the
57
+ * {@link ReleaseIterator#hasNext()} and {@link ReleaseIterator#next()} methods all obtain this object's monitor
58
+ * ({@code synchronized}). This may cause one iterator having to wait for another iterator's implicit loading actions.
59
+ * <p>
37 60
*
38 61
* @author Axel Uhl (d043530)
39 62
*/
... ...
@@ -63,48 +86,13 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
63 86
* reached the oldest release in the cache, iteration is complete, and no further page loading is necessary
64 87
* to complete the iteration.
65 88
*/
66
- private final TreeMap<TimePoint, Release> releasesByPublishingTimePoint;
89
+ private final ConcurrentNavigableMap<TimePoint, Release> releasesByPublishingTimePoint;
67 90
68 91
private boolean cacheContainsOldestRelease;
69 92
70 93
private TimePoint lastFetchOfNewestReleases;
71 94
72
- private final static Duration RELOAD_NEWEST_RELEASES_AFTER_DURATION = Duration.ONE_MINUTE;
73
-
74
- public GithubReleasesRepository(String owner, String repositoryName, String defaultReleaseNamePrefix) {
75
- super(defaultReleaseNamePrefix);
76
- this.owner = owner;
77
- this.repositoryName = repositoryName;
78
- this.releasesByPublishingTimePoint = new TreeMap<>();
79
- this.cacheContainsOldestRelease = false;
80
- this.lastFetchOfNewestReleases = null;
81
- }
82
-
83
- @Override
84
- public Release getLatestRelease(String releaseNamePrefix) {
85
- Release result = null;
86
- for (final Release release : this) { // invokes the iterator() method
87
- if (release.getBaseName().equals(releaseNamePrefix)) {
88
- result = release;
89
- break; // here we assume that releases are enumerated from newest to oldest
90
- }
91
- }
92
- return result;
93
- }
94
-
95
- private String getRepositoryPath() {
96
- return owner+"/"+repositoryName;
97
- }
98
-
99
- private String getReleasesURL() {
100
- return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100";
101
- }
102
-
103
- @Override
104
- public Release getRelease(String releaseName) {
105
- return new GithubRelease(releaseName, GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+releaseName+Release.ARCHIVE_EXTENSION,
106
- GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+Release.RELEASE_NOTES_FILE_NAME);
107
- }
95
+ private final static Duration RELOAD_NEWEST_RELEASES_AFTER_DURATION = Duration.ONE_MINUTE.times(2);
108 96
109 97
/**
110 98
* If {@link GithubReleasesRepository#lastFetchOfNewestReleases} is {@code null} or older than the
... ...
@@ -150,16 +138,21 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
150 138
private Iterator<Release> cachedReleasesIterator;
151 139
152 140
private ReleaseIterator() throws MalformedURLException, IOException, ParseException {
153
- nextPageURL = getReleasesURL();
154
- final TimePoint now = TimePoint.now();
155
- if (lastFetchOfNewestReleases != null && lastFetchOfNewestReleases.until(now)
156
- .compareTo(RELOAD_NEWEST_RELEASES_AFTER_DURATION) < 0) {
157
- cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
158
- } else {
159
- cachedReleasesIterator = null;
160
- while (nextPageURL != null && cachedReleasesIterator == null) {
161
- lastFetchOfNewestReleases = now;
162
- loadNextPage(/* olderThan */ null);
141
+ synchronized (GithubReleasesRepository.this) {
142
+ nextPageURL = getReleasesURL();
143
+ final TimePoint now = TimePoint.now();
144
+ if (lastFetchOfNewestReleases != null && lastFetchOfNewestReleases.until(now)
145
+ .compareTo(RELOAD_NEWEST_RELEASES_AFTER_DURATION) < 0) {
146
+ logger.fine(()->"No need to fetch page with newest releases; did that at "+lastFetchOfNewestReleases);
147
+ cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
148
+ } else {
149
+ logger.fine(()->"Need to fetch page with newest releases because last request was at "+
150
+ (lastFetchOfNewestReleases==null?"<never>":lastFetchOfNewestReleases));
151
+ cachedReleasesIterator = null;
152
+ while (nextPageURL != null && cachedReleasesIterator == null) {
153
+ lastFetchOfNewestReleases = now;
154
+ loadNextPage(/* olderThan */ null);
155
+ }
163 156
}
164 157
}
165 158
}
... ...
@@ -181,7 +174,8 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
181 174
* cache, and {@link #cachedReleasesIterator} is set to the newest element added to the cache, or set to
182 175
* {@code null} if no release was added to the cache by this call.
183 176
* <p>
184
- * Precondition: {@link #nextPageURL} is not {@code null}.
177
+ * Precondition: {@link #nextPageURL} is not {@code null}; and the calling thread owns the object monitor of
178
+ * the enclosing {@link GithubReleasesRepository} instance.
185 179
* <p>
186 180
* Postcondition: {@link GithubReleasesRepository#cacheContainsOldestRelease} is {@code true} if and only if
187 181
* this invocation has loaded the last page of releases that exist
... ...
@@ -194,11 +188,18 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
194 188
* to {@code null} if no releases older than {@code olderThan} were found during this invocation.
195 189
*/
196 190
private void loadNextPage(TimePoint olderThan) throws MalformedURLException, IOException, ParseException {
191
+ assert Thread.holdsLock(GithubReleasesRepository.this);
197 192
cachedReleasesIterator = null;
193
+ logger.info("Requesting releases page "+nextPageURL+(olderThan==null?"":(" looking for releases older than "+olderThan)));
198 194
final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(nextPageURL));
199 195
final InputStream index = (InputStream) connection.getContent();
196
+ final String xRatelimitRemaining = connection.getHeaderField("x-ratelimit-remaining");
197
+ if (xRatelimitRemaining != null && Integer.valueOf(xRatelimitRemaining) <= 0) {
198
+ throw new RuntimeException("You hit the rate limit of "+connection.getHeaderField("x-ratelimit-limit"));
199
+ }
200 200
final String linkHeader = connection.getHeaderField("link");
201 201
nextPageURL = getNextPageURL(linkHeader);
202
+ logger.fine(()->nextPageURL==null?"This was the last page":("Next page will be "+nextPageURL));
202 203
cacheContainsOldestRelease = cacheContainsOldestRelease || nextPageURL == null; // in this case we have seen and cached the last (oldest) page of releases
203 204
final JSONArray releasesJson = (JSONArray) new JSONParser().parse(new InputStreamReader(index));
204 205
boolean addedAtLeastOneReleaseToCache = false;
... ...
@@ -232,34 +233,40 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
232 233
233 234
@Override
234 235
public boolean hasNext() {
235
- // - we're delivering from the cache and the cache has more elements, or
236
- // - we've reached the end of the cache but the cache doesn't contain the oldest release and we can load more pages
237
- return cachedReleasesIterator != null && cachedReleasesIterator.hasNext()
238
- || !cacheContainsOldestRelease && nextPageURL != null;
236
+ synchronized (GithubReleasesRepository.this) {
237
+ // - we're delivering from the cache and the cache has more elements, or
238
+ // - we've reached the end of the cache but the cache doesn't contain the oldest release and we can load more pages
239
+ return cachedReleasesIterator != null && cachedReleasesIterator.hasNext()
240
+ || !cacheContainsOldestRelease && nextPageURL != null;
241
+ }
239 242
}
240 243
241 244
@Override
242 245
public Release next() {
243
- final Release result;
244
- if (cachedReleasesIterator != null && cachedReleasesIterator.hasNext()) {
245
- result = getNextElementFromCacheIterator();
246
- } else if (cacheContainsOldestRelease) {
247
- throw new NoSuchElementException();
248
- } else {
249
- while (nextPageURL != null && cachedReleasesIterator == null) {
250
- try {
251
- loadNextPage(/* olderThan */ releasesByPublishingTimePoint.firstKey());
252
- } catch (IOException | ParseException e) {
253
- throw new RuntimeException(e);
254
- }
255
- }
256
- if (cachedReleasesIterator == null || !cachedReleasesIterator.hasNext()) {
257
- throw new NoSuchElementException();
258
- } else {
246
+ synchronized (GithubReleasesRepository.this) {
247
+ final Release result;
248
+ if (cachedReleasesIterator != null && cachedReleasesIterator.hasNext()) {
259 249
result = getNextElementFromCacheIterator();
250
+ } else {
251
+ if (cacheContainsOldestRelease) {
252
+ throw new NoSuchElementException();
253
+ } else {
254
+ while (nextPageURL != null && cachedReleasesIterator == null) {
255
+ try {
256
+ loadNextPage(/* olderThan */ releasesByPublishingTimePoint.firstKey());
257
+ } catch (IOException | ParseException e) {
258
+ throw new RuntimeException(e);
259
+ }
260
+ }
261
+ if (cachedReleasesIterator == null || !cachedReleasesIterator.hasNext()) {
262
+ throw new NoSuchElementException();
263
+ } else {
264
+ result = getNextElementFromCacheIterator();
265
+ }
266
+ }
260 267
}
268
+ return result;
261 269
}
262
- return result;
263 270
}
264 271
265 272
private Release getNextElementFromCacheIterator() {
... ...
@@ -272,6 +279,41 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
272 279
}
273 280
}
274 281
282
+ public GithubReleasesRepository(String owner, String repositoryName, String defaultReleaseNamePrefix) {
283
+ super(defaultReleaseNamePrefix);
284
+ this.owner = owner;
285
+ this.repositoryName = repositoryName;
286
+ this.releasesByPublishingTimePoint = new ConcurrentSkipListMap<>();
287
+ this.cacheContainsOldestRelease = false;
288
+ this.lastFetchOfNewestReleases = null;
289
+ }
290
+
291
+ @Override
292
+ public Release getLatestRelease(String releaseNamePrefix) {
293
+ Release result = null;
294
+ for (final Release release : this) { // invokes the iterator() method
295
+ if (release.getBaseName().equals(releaseNamePrefix)) {
296
+ result = release;
297
+ break; // here we assume that releases are enumerated from newest to oldest
298
+ }
299
+ }
300
+ return result;
301
+ }
302
+
303
+ private String getRepositoryPath() {
304
+ return owner+"/"+repositoryName;
305
+ }
306
+
307
+ private String getReleasesURL() {
308
+ return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100";
309
+ }
310
+
311
+ @Override
312
+ public Release getRelease(String releaseName) {
313
+ return new GithubRelease(releaseName, GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+releaseName+Release.ARCHIVE_EXTENSION,
314
+ GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+Release.RELEASE_NOTES_FILE_NAME);
315
+ }
316
+
275 317
@Override
276 318
public Iterator<Release> iterator() {
277 319
try {
java/target/configuration/logging_debug.properties
... ...
@@ -61,4 +61,7 @@ com.sap.sailing.domain.queclinkadapter.tracker.QueclinkUDPTracker.level = FINE
61 61
# Show locking progress in AIAgentImpl:
62 62
#com.sap.sailing.aiagent.impl.AIAgentImpl.level = FINE
63 63
# Show AI rules task enqueuing:
64
-com.sap.sailing.aiagent.impl.RaceListener.level = FINE
... ...
\ No newline at end of file
0
+com.sap.sailing.aiagent.impl.RaceListener.level = FINE
1
+
2
+# Show GithubReleasesRepository log output
3
+com.sap.sse.landscape.impl.GithubReleasesRepository.level = FINE
... ...
\ No newline at end of file