summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaurice Laveaux <maurice@localhost.localdomain>2014-05-26 16:49:06 +0200
committerMaurice Laveaux <maurice@localhost.localdomain>2014-05-26 16:49:06 +0200
commit0ee51c21a1cf255cc39802e4e57d4f86ee5b2330 (patch)
tree4d5c2a2d1550dcd0d95fb5365a18a0831823f6dd
parent1a44e63644fb0809c339fe59ea47e15d1663b997 (diff)
downloadTwitterDataAnalytics-0ee51c21a1cf255cc39802e4e57d4f86ee5b2330.tar.gz
Improved the requester
* Reads a gzip stream. * Uses the http request date for interval measurement.
-rw-r--r--src/io/AbstractRequester.java30
-rw-r--r--src/io/Response.java11
2 files changed, 28 insertions, 13 deletions
diff --git a/src/io/AbstractRequester.java b/src/io/AbstractRequester.java
index 913e9bd..333e39d 100644
--- a/src/io/AbstractRequester.java
+++ b/src/io/AbstractRequester.java
@@ -13,6 +13,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
@@ -30,15 +31,16 @@ public abstract class AbstractRequester implements Requester {
throws IOException, RateLimitException {
HttpURLConnection conn = open(buildUrl(resource));
try {
+ conn.addRequestProperty("Accept-Encoding", "gzip");
preconnect(conn);
+
JsonElement resp = getResponseAsJson(conn);
if (resp.isJsonObject() && resp.getAsJsonObject().has("errors")) {
/* print response to stderr for debugging */
String errors = resp.getAsJsonObject().get("errors").toString();
- getLogger().log(Level.INFO, "Request failed: {0}", errors);
+ getLogger().info("Request " + resource + " failed: " + errors);
}
-
// TODO: what if there is an internal server error? Technically we
// should always treat that as "don't know what the server thinks".
@@ -48,14 +50,16 @@ public abstract class AbstractRequester implements Requester {
throw new RateLimitException(Integer.parseInt(conn.getHeaderField("X-Rate-Limit-Reset")));
}
// TODO: print more helpful details
- throw new IOException("Unexpected response code");
+ throw new IOException("Unexpected response code "
+ + conn.getResponseCode() + " for " + resource);
}
-
+
int rateLimit = Integer.parseInt(conn.getHeaderField("X-Rate-Limit-Limit"));
int rateLimitRemaining = Integer.parseInt(conn.getHeaderField("X-Rate-Limit-Remaining"));
- int rateLimitReset = Integer.parseInt(conn.getHeaderField("X-Rate-Limit-Reset"));
+ long rateLimitReset = Long.parseLong(conn.getHeaderField("X-Rate-Limit-Reset"));
+
+ return new Response(resp, rateLimit, rateLimitRemaining, rateLimitReset, rateLimitReset * 1000 - conn.getDate());
- return new Response(resp, rateLimit, rateLimitRemaining, rateLimitReset);
} finally {
conn.disconnect();
}
@@ -115,10 +119,10 @@ public abstract class AbstractRequester implements Requester {
StringWriter writer = new StringWriter();
InputStream is;
try {
- is = conn.getInputStream();
- } catch (FileNotFoundException ex) {
+ is = wrapGZip(conn, conn.getInputStream());
+ } catch (IOException ex) {
/* 404 (FileNotFoundException) */
- is = conn.getErrorStream();
+ is = wrapGZip(conn, conn.getErrorStream());
}
// this could happen if the URL was severly malformed
if (is == null) {
@@ -130,6 +134,14 @@ public abstract class AbstractRequester implements Requester {
return parser.parse(writer.toString());
}
+ private InputStream wrapGZip(HttpURLConnection conn, InputStream in) throws IOException {
+ if ("gzip".equals(conn.getContentEncoding())) {
+ return new GZIPInputStream(in);
+ } else {
+ return conn.getInputStream();
+ }
+ }
+
/**
* Prepare the request before it gets send.
*
diff --git a/src/io/Response.java b/src/io/Response.java
index e5ede8a..b2681a8 100644
--- a/src/io/Response.java
+++ b/src/io/Response.java
@@ -11,17 +11,20 @@ public class Response {
private final JsonElement element;
- private final int rateLimitReset;
+ private final long rateLimitReset;
+
+ private final long timeRemaining;
private final int rateLimitRemaining;
private final int rateLimit;
- public Response(JsonElement resp, int rateLimit, int rateLimitRemaining, int rateLimitReset) {
+ public Response(JsonElement resp, int rateLimit, int rateLimitRemaining, long rateLimitReset, long timeRemaining) {
this.element = resp;
this.rateLimit = rateLimit;
this.rateLimitRemaining = rateLimitRemaining;
this.rateLimitReset = rateLimitReset;
+ this.timeRemaining = timeRemaining;
}
public JsonElement getJson() {
@@ -36,7 +39,7 @@ public class Response {
return this.rateLimitRemaining;
}
- public int getRateLimitReset() {
+ public long getRateLimitReset() {
return this.rateLimitReset;
}
@@ -45,6 +48,6 @@ public class Response {
* similar request can be executed.
*/
public long getRateLimitRemainingTime() {
- return rateLimitReset * 1000 - System.currentTimeMillis();
+ return this.timeRemaining;
}
}