java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/AbstractReleaseRepository.java
... ...
@@ -1,7 +1,5 @@
1 1
package com.sap.sse.landscape.impl;
2 2
3
-import java.util.Iterator;
4
-
5 3
import com.sap.sse.common.Util;
6 4
import com.sap.sse.landscape.Release;
7 5
import com.sap.sse.landscape.ReleaseRepository;
... ...
@@ -19,17 +17,10 @@ public abstract class AbstractReleaseRepository implements ReleaseRepository {
19 17
return defaultReleaseNamePrefix;
20 18
}
21 19
22
- protected abstract Iterable<Release> getAvailableReleases();
23
-
24
- @Override
25
- public Iterator<Release> iterator() {
26
- return getAvailableReleases().iterator();
27
- }
28
-
29 20
@Override
30 21
public Release getLatestRelease(String releaseNamePrefix) {
31 22
Release result = null;
32
- for (final Release release : getAvailableReleases()) {
23
+ for (final Release release : this) { // invokes the iterator() method
33 24
if (release.getBaseName().equals(releaseNamePrefix) &&
34 25
(result == null || release.getCreationDate().after(result.getCreationDate()))) {
35 26
result = release;
... ...
@@ -40,6 +31,6 @@ public abstract class AbstractReleaseRepository implements ReleaseRepository {
40 31
41 32
@Override
42 33
public Release getRelease(String releaseName) {
43
- return Util.first(Util.filter(getAvailableReleases(), r->r.getName().equals(releaseName)));
34
+ return Util.first(Util.filter(this, r->r.getName().equals(releaseName)));
44 35
}
45 36
}
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/FolderBasedReleaseRepositoryImpl.java
... ...
@@ -6,6 +6,7 @@ import java.io.InputStream;
6 6
import java.net.URL;
7 7
import java.net.URLConnection;
8 8
import java.util.Collections;
9
+import java.util.Iterator;
9 10
import java.util.LinkedList;
10 11
import java.util.List;
11 12
import java.util.logging.Level;
... ...
@@ -39,7 +40,12 @@ public class FolderBasedReleaseRepositoryImpl extends AbstractReleaseRepository
39 40
return repositoryBase;
40 41
}
41 42
42
- protected Iterable<Release> getAvailableReleases() {
43
+ @Override
44
+ public Iterator<Release> iterator() {
45
+ return getAvailableReleases().iterator();
46
+ }
47
+
48
+ private Iterable<Release> getAvailableReleases() {
43 49
final List<Release> result = new LinkedList<>();
44 50
try {
45 51
final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(getRepositoryBase()));
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/GithubReleasesRepository.java
... ...
@@ -3,10 +3,14 @@ package com.sap.sse.landscape.impl;
3 3
import java.io.IOException;
4 4
import java.io.InputStream;
5 5
import java.io.InputStreamReader;
6
+import java.net.MalformedURLException;
6 7
import java.net.URL;
7 8
import java.net.URLConnection;
9
+import java.text.SimpleDateFormat;
10
+import java.util.Iterator;
8 11
import java.util.LinkedList;
9 12
import java.util.List;
13
+import java.util.TreeMap;
10 14
import java.util.logging.Logger;
11 15
import java.util.regex.Matcher;
12 16
import java.util.regex.Pattern;
... ...
@@ -16,27 +20,35 @@ import org.json.simple.JSONObject;
16 20
import org.json.simple.parser.JSONParser;
17 21
import org.json.simple.parser.ParseException;
18 22
23
+import com.sap.sse.common.TimePoint;
24
+import com.sap.sse.common.Util.Pair;
19 25
import com.sap.sse.landscape.Release;
20 26
import com.sap.sse.landscape.ReleaseRepository;
21 27
import com.sap.sse.util.HttpUrlConnectionHelper;
22 28
23 29
/**
24 30
* Assumes a public GitHub repository where releases can be freely downloaded from
25
- * <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>.
31
+ * <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>. The GitHub
32
+ * {@code /releases} end point delivers the releases in descending chronological order, so
33
+ * newest releases first. With this, we can cache old results and try to get along with the
34
+ * harsh rate limit of only 60 requests per hour when used without authentication.
26 35
*
27 36
* @author Axel Uhl (d043530)
28 37
*/
29 38
public class GithubReleasesRepository extends AbstractReleaseRepository implements ReleaseRepository {
30 39
private final static Logger logger = Logger.getLogger(GithubReleasesRepository.class.getName());
40
+ private static final SimpleDateFormat isoDateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
31 41
private final static String GITHUB_API_BASE_URL = "https://api.github.com";
32 42
private final static String GITHUB_BASE_URL = "https://github.com";
33 43
private final String owner;
34 44
private final String repositoryName;
45
+ private final TreeMap<TimePoint, Release> releasesByPublishingTimePoint;
35 46
36 47
public GithubReleasesRepository(String owner, String repositoryName, String defaultReleaseNamePrefix) {
37 48
super(defaultReleaseNamePrefix);
38 49
this.owner = owner;
39 50
this.repositoryName = repositoryName;
51
+ this.releasesByPublishingTimePoint = new TreeMap<>();
40 52
}
41 53
42 54
private String getRepositoryPath() {
... ...
@@ -44,7 +56,7 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
44 56
}
45 57
46 58
private String getReleasesURL() {
47
- return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100"; // TODO for unauthenticated requests there is a harsh rate limit of 60 requests per hour...
59
+ return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100";
48 60
}
49 61
50 62
@Override
... ...
@@ -52,9 +64,84 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
52 64
return new GithubRelease(releaseName, GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+releaseName+Release.ARCHIVE_EXTENSION,
53 65
GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+Release.RELEASE_NOTES_FILE_NAME);
54 66
}
67
+
68
+ /**
69
+ * Always fetches the first page from the {@code /releases} end point and starts constructing releases, until a
70
+ * publishing time point overlap with {@link GithubReleasesRepository#releasesByPublishingTimePoint} is found. Then
71
+ * we know we can continue to enumerate the remaining releases from that cache.
72
+ * <p>
73
+ *
74
+ * All releases found by loading a page are added to the
75
+ * {@link GithubReleasesRepository#releasesByPublishingTimePoint} cache.
76
+ *
77
+ * @author Axel Uhl (d043530)
78
+ *
79
+ */
80
+ private class ReleaseIterator implements Iterator<Release> {
81
+ private String nextPageURL;
82
+ private Iterator<Pair<TimePoint, GithubRelease>> publishingTimePointsAndReleasesFromCurrentPageIterator;
83
+
84
+ private ReleaseIterator() throws MalformedURLException, IOException, ParseException {
85
+ nextPageURL = getReleasesURL();
86
+ loadNextPage();
87
+ }
88
+
89
+ private void loadNextPage() throws MalformedURLException, IOException, ParseException {
90
+ final List<Pair<TimePoint, GithubRelease>> result = new LinkedList<>();
91
+ final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(nextPageURL));
92
+ final InputStream index = (InputStream) connection.getContent();
93
+ final String linkHeader = connection.getHeaderField("link");
94
+ nextPageURL = getNextPageURL(linkHeader);
95
+ final JSONArray releasesJson = (JSONArray) new JSONParser().parse(new InputStreamReader(index));
96
+ for (final Object releaseObject : releasesJson) {
97
+ final Pair<TimePoint, GithubRelease> publishedAtAndRelease = getPublishedAtAndReleaseFromJson((JSONObject) releaseObject);
98
+ releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
99
+ result.add(publishedAtAndRelease);
100
+ }
101
+ publishingTimePointsAndReleasesFromCurrentPageIterator = result.iterator();
102
+ }
103
+
104
+ @Override
105
+ public boolean hasNext() {
106
+ return publishingTimePointsAndReleasesFromCurrentPageIterator.hasNext() || nextPageURL != null;
107
+ }
108
+
109
+ @Override
110
+ public Release next() {
111
+ if (!publishingTimePointsAndReleasesFromCurrentPageIterator.hasNext()) {
112
+ try {
113
+ // FIXME bug6173: only load next page if we have to... we may already have created an overlap with the cache from releasesByPublishingTimePoint
114
+ loadNextPage();
115
+ } catch (IOException | ParseException e) {
116
+ throw new RuntimeException(e);
117
+ }
118
+ }
119
+ return publishingTimePointsAndReleasesFromCurrentPageIterator.next().getB();
120
+ }
121
+ }
122
+
123
+ @Override
124
+ public Iterator<Release> iterator() {
125
+ try {
126
+ return new ReleaseIterator();
127
+ } catch (Exception e) {
128
+ throw new RuntimeException(e);
129
+ }
130
+ }
55 131
56 132
@Override
57
- protected Iterable<Release> getAvailableReleases() {
133
+ public Release getLatestRelease(String releaseNamePrefix) {
134
+ // TODO Auto-generated method stub
135
+ return super.getLatestRelease(releaseNamePrefix);
136
+ }
137
+
138
+ /**
139
+ * Enumerating all releases of the GitHub repo is possible but goes against the harsh rate limit when used without
140
+ * an access token (currently only 60 requests per hour), so should ideally be avoided altogether. And if it is ever called,
141
+ * we will cache the results, so that for later requests we typically need to query only a single page, delivering the latest
142
+ * additions, if any.
143
+ */
144
+ private Iterable<Release> getAvailableReleases() {
58 145
final List<Release> result = new LinkedList<>();
59 146
try {
60 147
String nextPageURL = getReleasesURL();
... ...
@@ -74,20 +161,34 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
74 161
75 162
private void addAllReleasesTo(JSONArray releasesJson, List<Release> result) {
76 163
for (final Object releaseObject : releasesJson) {
77
- final JSONObject releaseJson = (JSONObject) releaseObject;
78
- final String name = releaseJson.get("name").toString();
79
- String archiveDownloadURL = null;
80
- String releaseNotesURL = null;
81
- for (final Object archiveAsset : (JSONArray) releaseJson.get("assets")) {
82
- final JSONObject archiveAssetJson = (JSONObject) archiveAsset;
83
- if (archiveAssetJson.get("content_type").equals("application/x-tar")) {
84
- archiveDownloadURL = archiveAssetJson.get("browser_download_url").toString();
85
- } else if (archiveAssetJson.get("name").equals(Release.RELEASE_NOTES_FILE_NAME)) {
86
- releaseNotesURL = archiveAssetJson.get("browser_download_url").toString();
87
- }
164
+ final Pair<TimePoint, GithubRelease> publishedAtAndRelease = getPublishedAtAndReleaseFromJson((JSONObject) releaseObject);
165
+ result.add(publishedAtAndRelease.getB());
166
+ releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
167
+ }
168
+ }
169
+
170
+ private Pair<TimePoint, GithubRelease> getPublishedAtAndReleaseFromJson(JSONObject releaseJson) {
171
+ final String name = releaseJson.get("name").toString();
172
+ final String publishedAtISO = releaseJson.get("published_at").toString();
173
+ TimePoint publishedAt;
174
+ try {
175
+ publishedAt = TimePoint.of(isoDateTimeFormat.parse(publishedAtISO));
176
+ } catch (java.text.ParseException e) {
177
+ logger.warning("Couldn't read published_at time stamp for release "+name+": "+publishedAtISO);
178
+ throw new RuntimeException(e);
179
+ }
180
+ String archiveDownloadURL = null;
181
+ String releaseNotesURL = null;
182
+ for (final Object archiveAsset : (JSONArray) releaseJson.get("assets")) {
183
+ final JSONObject archiveAssetJson = (JSONObject) archiveAsset;
184
+ if (archiveAssetJson.get("content_type").equals("application/x-tar")) {
185
+ archiveDownloadURL = archiveAssetJson.get("browser_download_url").toString();
186
+ } else if (archiveAssetJson.get("name").equals(Release.RELEASE_NOTES_FILE_NAME)) {
187
+ releaseNotesURL = archiveAssetJson.get("browser_download_url").toString();
88 188
}
89
- result.add(new GithubRelease(name, archiveDownloadURL, releaseNotesURL));
90 189
}
190
+ final GithubRelease release = new GithubRelease(name, archiveDownloadURL, releaseNotesURL);
191
+ return new Pair<>(publishedAt, release);
91 192
}
92 193
93 194
private static final Pattern nextPagePattern = Pattern.compile(".*<([^<]*)>; rel=\"next\".*");