java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/GithubReleasesRepository.java
... ...
@@ -8,6 +8,8 @@ import java.net.URL;
8 8
import java.net.URLConnection;
9 9
import java.text.SimpleDateFormat;
10 10
import java.util.Iterator;
11
+import java.util.LinkedList;
12
+import java.util.List;
11 13
import java.util.NoSuchElementException;
12 14
import java.util.concurrent.ConcurrentNavigableMap;
13 15
import java.util.concurrent.ConcurrentSkipListMap;
... ...
@@ -65,6 +67,7 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
65 67
private static final SimpleDateFormat isoDateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
66 68
private final static String GITHUB_API_BASE_URL = "https://api.github.com";
67 69
private final static String GITHUB_BASE_URL = "https://github.com";
70
+ private final static int NUMBER_OF_RELEASES_PER_PAGE = 100; // default would be 30; maximum is 100
68 71
private final String owner;
69 72
private final String repositoryName;
70 73
... ...
@@ -90,6 +93,19 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
90 93
91 94
private boolean cacheContainsOldestRelease;
92 95
96
+ /**
97
+ * If {@link #cacheContainsOldestRelease} is {@code false}, meaning the cache hasn't seen all releases back to the oldest
98
+ * one, this field holds the URL of the next page to load that will obtain older releases. Note that if newer releases
99
+ * have been published since loading the so far oldest cached releases, one or more newer releases may have slipped
100
+ * into that next page. But page loading of older releases will add only releases effectively older than the oldest
101
+ * release in the cache so far.<p>
102
+ *
103
+ * It starts out with {@link #getReleasesURL()}.<p>
104
+ *
105
+ * Will be {@code null} if {@link #cacheContainsOldestRelease} is {@code true}.
106
+ */
107
+ private String nextPageURLForOlderReleases;
108
+
93 109
private TimePoint lastFetchOfNewestReleases;
94 110
95 111
private final static Duration RELOAD_NEWEST_RELEASES_AFTER_DURATION = Duration.ONE_MINUTE.times(2);
... ...
@@ -123,121 +139,33 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
123 139
*/
124 140
private class ReleaseIterator implements Iterator<Release> {
125 141
/**
126
- * Initialized to the URL for loading the first page of releases; each call to
127
- * {@link #loadNextPage(TimePoint)} changes this to the next page, or {@code null}
128
- * if the last page was loaded.
129
- */
130
- private String nextPageURL;
131
-
132
- /**
133
- * Takes precedence if not {@code null} and still having elements; enumerates the cached releases, starting from
134
- * the newest (last in the cache) to the oldest (first in the cache). When fully consumed, page loading has to
135
- * continue until releases published earlier than the oldest one from the
136
- * {@link GithubReleasesRepository#releasesByPublishingTimePoint cache} are found.
142
+ * Used to enumerate the cached elements; will get assigned a new iterator after having reached the "old" end of
143
+ * the cache and {@link GithubReleasesRepository#loadPageWithNextOlderReleases() loading more older releases}.
137 144
*/
138 145
private Iterator<Release> cachedReleasesIterator;
139 146
140 147
private ReleaseIterator() throws MalformedURLException, IOException, ParseException {
141 148
synchronized (GithubReleasesRepository.this) {
142
- nextPageURL = getReleasesURL();
143 149
final TimePoint now = TimePoint.now();
144 150
if (lastFetchOfNewestReleases != null && lastFetchOfNewestReleases.until(now)
145 151
.compareTo(RELOAD_NEWEST_RELEASES_AFTER_DURATION) < 0) {
146 152
logger.fine(()->"No need to fetch page with newest releases; did that at "+lastFetchOfNewestReleases);
147
- cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
148 153
} else {
149 154
logger.fine(()->"Need to fetch page with newest releases because last request was at "+
150 155
(lastFetchOfNewestReleases==null?"<never>":lastFetchOfNewestReleases));
151
- cachedReleasesIterator = null;
152
- while (nextPageURL != null && cachedReleasesIterator == null) {
153
- lastFetchOfNewestReleases = now;
154
- loadNextPage(/* olderThan */ null);
155
- }
156
+ lastFetchOfNewestReleases = now;
157
+ fillCacheWithNewestReleases();
156 158
}
159
+ cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
157 160
}
158 161
}
159 162
160
- /**
161
- * Loads the page of releases referenced by {@link #nextPageURL}.
162
- * <p>
163
- *
164
- * If {@code olderThan} is {@code null}, only the releases newer than the newest entry in the cache are loaded
165
- * into the cache, and {@link #cachedReleasesIterator} is set to the newest element in the cache if and only if
166
- * the cache was empty when this method was called, or the page contained a release not newer than the newest
167
- * release in the cache. This also means that if with {@code olderThan==null} the
168
- * {@link #cachedReleasesIterator} is {@code null} after this method returns, one or more calls will be required
169
- * to create an "overlap" with the cache before starting the iteration. This is required because we guarantee
170
- * the cache to be "contiguous" in terms of the releases that exist.
171
- * <p>
172
- *
173
- * If {@code olderThan} is not {@code null}, only releases published before {@code olderThan} are added to the
174
- * cache, and {@link #cachedReleasesIterator} is set to the newest element added to the cache, or set to
175
- * {@code null} if no release was added to the cache by this call.
176
- * <p>
177
- * Precondition: {@link #nextPageURL} is not {@code null}; and the calling thread owns the object monitor of
178
- * the enclosing {@link GithubReleasesRepository} instance.
179
- * <p>
180
- * Postcondition: {@link GithubReleasesRepository#cacheContainsOldestRelease} is {@code true} if and only if
181
- * this invocation has loaded the last page of releases that exist
182
- *
183
- * @param olderThan
184
- * if {@code null}, releases newer than the newest release from the cache will be added to the cache,
185
- * and the {@link #cachedReleasesIterator} will be set to the then newest cache element; if not
186
- * {@code null}, only releases published before {@code olderThan} will be loaded, and
187
- * {@link #cachedReleasesIterator} is then set to the newest of the older releases loaded, if any, or
188
- * to {@code null} if no releases older than {@code olderThan} were found during this invocation.
189
- */
190
- private void loadNextPage(TimePoint olderThan) throws MalformedURLException, IOException, ParseException {
191
- assert Thread.holdsLock(GithubReleasesRepository.this);
192
- cachedReleasesIterator = null;
193
- logger.info("Requesting releases page "+nextPageURL+(olderThan==null?"":(" looking for releases older than "+olderThan)));
194
- final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(nextPageURL));
195
- final InputStream index = (InputStream) connection.getContent();
196
- final String xRatelimitRemaining = connection.getHeaderField("x-ratelimit-remaining");
197
- if (xRatelimitRemaining != null && Integer.valueOf(xRatelimitRemaining) <= 0) {
198
- throw new RuntimeException("You hit the rate limit of "+connection.getHeaderField("x-ratelimit-limit"));
199
- }
200
- final String linkHeader = connection.getHeaderField("link");
201
- nextPageURL = getNextPageURL(linkHeader);
202
- logger.fine(()->nextPageURL==null?"This was the last page":("Next page will be "+nextPageURL));
203
- cacheContainsOldestRelease = cacheContainsOldestRelease || nextPageURL == null; // in this case we have seen and cached the last (oldest) page of releases
204
- final JSONArray releasesJson = (JSONArray) new JSONParser().parse(new InputStreamReader(index));
205
- boolean addedAtLeastOneReleaseToCache = false;
206
- final boolean cacheWasEmpty = releasesByPublishingTimePoint.isEmpty();
207
- for (final Object releaseObject : releasesJson) {
208
- final Pair<TimePoint, GithubRelease> publishedAtAndRelease = getPublishedAtAndReleaseFromJson((JSONObject) releaseObject);
209
- if (olderThan == null) { // looking for releases published after the newest cache entry
210
- if (cacheWasEmpty || publishedAtAndRelease.getA().after(releasesByPublishingTimePoint.lastKey())) {
211
- addedAtLeastOneReleaseToCache = true;
212
- releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
213
- } else {
214
- cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
215
- }
216
- } else { // looking for releases published before olderThan
217
- if (publishedAtAndRelease.getA().before(olderThan)) {
218
- addedAtLeastOneReleaseToCache = true;
219
- releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
220
- }
221
- }
222
- }
223
- if (olderThan == null) {
224
- if (cacheWasEmpty) {
225
- cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
226
- }
227
- } else {
228
- if (addedAtLeastOneReleaseToCache) {
229
- cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().tailMap(olderThan, /* inclusive */ false).values().iterator();
230
- }
231
- }
232
- }
233
-
234 163
@Override
235 164
public boolean hasNext() {
236 165
synchronized (GithubReleasesRepository.this) {
237 166
// - we're delivering from the cache and the cache has more elements, or
238
- // - we've reached the end of the cache but the cache doesn't contain the oldest release and we can load more pages
239
- return cachedReleasesIterator != null && cachedReleasesIterator.hasNext()
240
- || !cacheContainsOldestRelease && nextPageURL != null;
167
+ // - we've reached the end of the cache but the cache doesn't contain the oldest release so we can load more pages
168
+ return cachedReleasesIterator.hasNext() || !cacheContainsOldestRelease;
241 169
}
242 170
}
243 171
... ...
@@ -245,38 +173,29 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
245 173
public Release next() {
246 174
synchronized (GithubReleasesRepository.this) {
247 175
final Release result;
248
- if (cachedReleasesIterator != null && cachedReleasesIterator.hasNext()) {
249
- result = getNextElementFromCacheIterator();
176
+ if (cachedReleasesIterator.hasNext()) {
177
+ result = cachedReleasesIterator.next();
250 178
} else {
251 179
if (cacheContainsOldestRelease) {
252 180
throw new NoSuchElementException();
253 181
} else {
254
- while (nextPageURL != null && cachedReleasesIterator == null) {
255
- try {
256
- loadNextPage(/* olderThan */ releasesByPublishingTimePoint.firstKey());
257
- } catch (IOException | ParseException e) {
258
- throw new RuntimeException(e);
259
- }
182
+ final TimePoint oldestReleaseSoFar = releasesByPublishingTimePoint.firstKey();
183
+ try {
184
+ loadPageWithNextOlderReleases();
185
+ } catch (IOException | ParseException e) {
186
+ throw new RuntimeException(e);
260 187
}
261
- if (cachedReleasesIterator == null || !cachedReleasesIterator.hasNext()) {
188
+ cachedReleasesIterator = releasesByPublishingTimePoint.headMap(oldestReleaseSoFar).descendingMap().values().iterator();
189
+ if (!cachedReleasesIterator.hasNext()) {
262 190
throw new NoSuchElementException();
263 191
} else {
264
- result = getNextElementFromCacheIterator();
192
+ result = cachedReleasesIterator.next();
265 193
}
266 194
}
267 195
}
268 196
return result;
269 197
}
270 198
}
271
-
272
- private Release getNextElementFromCacheIterator() {
273
- final Release result;
274
- result = cachedReleasesIterator.next();
275
- if (!cachedReleasesIterator.hasNext()) {
276
- cachedReleasesIterator = null;
277
- }
278
- return result;
279
- }
280 199
}
281 200
282 201
public GithubReleasesRepository(String owner, String repositoryName, String defaultReleaseNamePrefix) {
... ...
@@ -285,9 +204,106 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
285 204
this.repositoryName = repositoryName;
286 205
this.releasesByPublishingTimePoint = new ConcurrentSkipListMap<>();
287 206
this.cacheContainsOldestRelease = false;
207
+ this.nextPageURLForOlderReleases = getReleasesURL();
288 208
this.lastFetchOfNewestReleases = null;
289 209
}
290 210
211
+ /**
212
+ * Loads a page of releases from {@code pageURL} and returns the link to the next page, or {@code null} if this was
213
+ * the last page available (therefore containing the oldest releases). The releases loaded are added to the cache if
214
+ * they are outside of the contiguous time range between oldest and newest publishing date as it was found in the
215
+ * cache when this method is invoked.
216
+ * <p>
217
+ *
218
+ * The method makes no changes to the cache or any other state of this instance.
219
+ *
220
+ * @return the link to the next page in the returned pair's {@link Pair#getA() A component}, and the sequence of
221
+ * releases loaded from that page with their publishing time points, ordered from newest to oldest.
222
+ */
223
+ private synchronized Pair<String, Iterable<Pair<TimePoint, GithubRelease>>> getReleasesFromPage(String pageURL) throws IOException, ParseException {
224
+ logger.info("Requesting releases page "+pageURL);
225
+ final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(pageURL));
226
+ final InputStream index = (InputStream) connection.getContent();
227
+ final String xRatelimitRemaining = connection.getHeaderField("x-ratelimit-remaining");
228
+ logger.fine(()->""+xRatelimitRemaining+" requests left in this hour");
229
+ if (xRatelimitRemaining != null && Integer.valueOf(xRatelimitRemaining) <= 0) {
230
+ throw new RuntimeException("You hit the rate limit of "+connection.getHeaderField("x-ratelimit-limit"));
231
+ }
232
+ final String linkHeader = connection.getHeaderField("link");
233
+ final String nextPageURL = getNextPageURL(linkHeader);
234
+ logger.fine(()->nextPageURL==null?"This was the last page":("Next page will be "+nextPageURL));
235
+ final List<Pair<TimePoint, GithubRelease>> publishingTimePointsAndReleases = new LinkedList<>();
236
+ final JSONArray releasesJson = (JSONArray) new JSONParser().parse(new InputStreamReader(index));
237
+ for (final Object releaseObject : releasesJson) {
238
+ publishingTimePointsAndReleases.add(getPublishedAtAndReleaseFromJson((JSONObject) releaseObject));
239
+ }
240
+ return new Pair<>(nextPageURL, publishingTimePointsAndReleases);
241
+ }
242
+
243
+ /**
244
+ * Loads at least one page of releases, starting with {@link #getReleasesURL()}. The method ensures that a time-wise
245
+ * overlap with any pre-existing cache entries is established so that the cache entries are a contiguous prefix of
246
+ * the list of all releases that exist in the repository.
247
+ * <p>
248
+ *
249
+ * In particular, if the cache is empty when this method is called, only the first page needs loading.
250
+ * <p>
251
+ *
252
+ * If the cache already contained one or more releases when this method is called, page loading continues
253
+ * with the next page until a page contains a release published not after the newest release in the cache
254
+ * at the time when this method was called.
255
+ */
256
+ private synchronized void fillCacheWithNewestReleases() throws IOException, ParseException {
257
+ final boolean cacheWasEmpty = releasesByPublishingTimePoint.isEmpty();
258
+ final TimePoint publishingTimePointOfLatestReleaseSoFar = cacheWasEmpty ? null : releasesByPublishingTimePoint.lastKey();
259
+ String nextPageURL = getReleasesURL();
260
+ boolean overlap = false;
261
+ do {
262
+ final Pair<String, Iterable<Pair<TimePoint, GithubRelease>>> pageResults = getReleasesFromPage(nextPageURL);
263
+ for (final Pair<TimePoint, GithubRelease> publishingTimePointAndRelease : pageResults.getB()) {
264
+ overlap = !cacheWasEmpty && !publishingTimePointAndRelease.getA().after(publishingTimePointOfLatestReleaseSoFar);
265
+ if (cacheWasEmpty || publishingTimePointAndRelease.getA().after(publishingTimePointOfLatestReleaseSoFar)) {
266
+ releasesByPublishingTimePoint.put(publishingTimePointAndRelease.getA(), publishingTimePointAndRelease.getB());
267
+ }
268
+ }
269
+ nextPageURL = pageResults.getA();
270
+ if (cacheWasEmpty) {
271
+ rememberNextPageForOlderReleases(nextPageURL);
272
+ }
273
+ } while (!cacheWasEmpty && !overlap);
274
+ }
275
+
276
+ private void rememberNextPageForOlderReleases(String nextPageURL) {
277
+ nextPageURLForOlderReleases = nextPageURL;
278
+ if (nextPageURL == null) {
279
+ cacheContainsOldestRelease = true;
280
+ }
281
+ }
282
+
283
+ /**
284
+ * Loads the page referenced by {@link #nextPageURLForOlderReleases}, adjusting that very field to point to either
285
+ * the next older page or {@code null}, and adjusting {@link #cacheContainsOldestRelease} accordingly. Releases
286
+ * older than the so far oldest release are added to the cache. If {@link #cacheContainsOldestRelease} is
287
+ * {@code false}, the process continues until at least one older release has been added to the cache.<p>
288
+ *
289
+ * Precondition: {@code !}{@link #cacheContainsOldestRelease} {@code && }{@link #nextPageURLForOlderReleases}{@code != null}
290
+ */
291
+ private synchronized void loadPageWithNextOlderReleases() throws IOException, ParseException {
292
+ assert !cacheContainsOldestRelease;
293
+ final TimePoint publishingTimePointOfOldestReleaseInCacheSoFar = releasesByPublishingTimePoint.firstKey();
294
+ boolean addedAtLeastOneReleaseToCache = false;
295
+ do {
296
+ final Pair<String, Iterable<Pair<TimePoint, GithubRelease>>> pageResults = getReleasesFromPage(nextPageURLForOlderReleases);
297
+ for (final Pair<TimePoint, GithubRelease> publishedAtAndRelease : pageResults.getB()) {
298
+ if (publishedAtAndRelease.getA().before(publishingTimePointOfOldestReleaseInCacheSoFar)) {
299
+ addedAtLeastOneReleaseToCache = true;
300
+ releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
301
+ }
302
+ }
303
+ rememberNextPageForOlderReleases(pageResults.getA());
304
+ } while (!addedAtLeastOneReleaseToCache && !cacheContainsOldestRelease);
305
+ }
306
+
291 307
@Override
292 308
public Release getLatestRelease(String releaseNamePrefix) {
293 309
Release result = null;
... ...
@@ -305,7 +321,7 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
305 321
}
306 322
307 323
private String getReleasesURL() {
308
- return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100";
324
+ return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page="+NUMBER_OF_RELEASES_PER_PAGE;
309 325
}
310 326
311 327
@Override