summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/io/DataWriter.java42
-rw-r--r--src/io/StreamImpl.java40
-rw-r--r--src/main/Main.java46
-rw-r--r--src/main/TweetCounter.java18
-rw-r--r--src/main/TweetShell.java27
-rw-r--r--src/provider/CompositeResultListener.java5
-rw-r--r--src/provider/ResultListener.java4
7 files changed, 97 insertions, 85 deletions
diff --git a/src/io/DataWriter.java b/src/io/DataWriter.java
index c1e1ce9..f3d4006 100644
--- a/src/io/DataWriter.java
+++ b/src/io/DataWriter.java
@@ -1,5 +1,8 @@
package io;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonParser;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.FileInputStream;
@@ -81,7 +84,7 @@ public class DataWriter implements ResultListener, Closeable, Flushable {
}
@Override
- public void tweetGenerated(JSONObject obj) {
+ public void tweetGenerated(JsonObject obj) {
try {
// ensure that the file is open
m_tweet.open();
@@ -105,19 +108,24 @@ public class DataWriter implements ResultListener, Closeable, Flushable {
try {
is = store.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+ JsonParser jsonParser = new JsonParser();
// parse each line into a JSONObject, read the id and add it to
// the set of ids.
while ((line = reader.readLine()) != null) {
- JSONObject obj = new JSONObject(line);
- long id = obj.getLong("id");
- idSet.add(id);
- lineno++;
+ try {
+ JsonObject obj = jsonParser.parse(line).getAsJsonObject();
+ long id = obj.get("id").getAsLong();
+ idSet.add(id);
+ lineno++;
+ } catch (JsonParseException ex) {
+ getLogger().log(Level.WARNING, "Tweet found without an id: {0}", ex);
+ }
}
} catch (FileNotFoundException ex) {
// ignore, file will be created if necessary.
- } catch (JSONException | IOException ex) {
+ } catch (IOException ex) {
if (line != null) {
- getLogger().log(Level.INFO, "Last line: " + line);
+ getLogger().log(Level.INFO, "Last line: {0}", line);
}
getLogger().log(Level.WARNING, store.getFileName()
+ ": error occurred in file at line " + lineno, ex);
@@ -134,21 +142,17 @@ public class DataWriter implements ResultListener, Closeable, Flushable {
* @param output The stream to write objects to.
* @param idSet The id set to add the obj id to.
*/
- private void writeObject(JSONObject obj, OutputStream output,
+ private void writeObject(JsonObject obj, OutputStream output,
Set<Long> idSet) {
- try {
- long id = obj.getLong("id");
+ long id = obj.get("id").getAsLong();
- if (!idSet.contains(id)) {
- try {
- output.write((obj.toString() + "\n").getBytes(Charsets.UTF_8));
- idSet.add(id);
- } catch (IOException ex) {
- getLogger().log(Level.WARNING, "Cannot write to file", ex);
- }
+ if (!idSet.contains(id)) {
+ try {
+ output.write((obj.toString() + "\n").getBytes(Charsets.UTF_8));
+ idSet.add(id);
+ } catch (IOException ex) {
+ getLogger().log(Level.WARNING, "Cannot write to file", ex);
}
- } catch (JSONException ex) {
- getLogger().log(Level.WARNING, "ID not found?!", ex);
}
}
diff --git a/src/io/StreamImpl.java b/src/io/StreamImpl.java
index a222bd8..b2f6b0c 100644
--- a/src/io/StreamImpl.java
+++ b/src/io/StreamImpl.java
@@ -1,5 +1,9 @@
package io;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonParser;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
@@ -13,15 +17,13 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Level;
import java.util.logging.Logger;
import mining.Stream;
import oauth.signpost.exception.OAuthException;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
import provider.ExceptionListener;
import provider.ResultListener;
import support.StreamingGZIPInputStream;
@@ -236,7 +238,7 @@ public class StreamImpl implements Stream {
// if the worker has any old ones left.
while (worker.isRunning() || worker.hasObjects()) {
try {
- JSONObject obj = worker.getObject();
+ JsonObject obj = worker.getObject();
if (obj == worker.endOfStream) {
// end of objects marker
assert !worker.isRunning();
@@ -249,7 +251,7 @@ public class StreamImpl implements Stream {
}
}
- private void processObject(JSONObject obj) {
+ private void processObject(JsonObject obj) {
synchronized (resultListenerSync) {
if (resultListener != null) {
resultListener.tweetGenerated(obj);
@@ -263,9 +265,9 @@ public class StreamImpl implements Stream {
private final String keywords;
private final HttpURLConnection connection;
private volatile boolean running = true;
- private final BlockingQueue<JSONObject> receivedObjects;
+ private final BlockingQueue<JsonObject> receivedObjects;
private InputStream inputStream;
- private final JSONObject endOfStream;
+ private final JsonObject endOfStream;
Worker(String keywords) throws IOException {
this.keywords = keywords;
@@ -280,7 +282,7 @@ public class StreamImpl implements Stream {
throw ex;
}
this.receivedObjects = new LinkedBlockingQueue<>();
- this.endOfStream = new JSONObject();
+ this.endOfStream = new JsonObject();
}
/**
@@ -382,17 +384,21 @@ public class StreamImpl implements Stream {
// See https://dev.twitter.com/docs/streaming-apis/messages
InputStreamReader isr = new InputStreamReader(is, Charsets.UTF_8);
BufferedReader reader = new BufferedReader(isr);
- JSONTokener jsonTokener = new JSONTokener(reader);
+ JsonParser parser = new JsonParser();
while (running) {
try {
- Object obj = jsonTokener.nextValue();
- if (obj instanceof JSONObject) {
- processReceivedObject((JSONObject) obj);
+ String line = reader.readLine();
+ if (line.isEmpty()) {
+ continue;
+ }
+ JsonElement elem = parser.parse(line);
+ if (elem.isJsonObject()) {
+ processReceivedObject(elem.getAsJsonObject());
} else {
- getLogger().severe("Got unexpected object: " + obj);
+ getLogger().log(Level.SEVERE, "Got unexpected object: {0}", elem);
throw new IOException("Got unexpected type from stream");
}
- } catch (JSONException ex) {
+ } catch (JsonParseException ex) {
// ignore IO errors for a stop request ("Socket closed")
if (running) {
throw new IOException(ex);
@@ -406,12 +412,12 @@ public class StreamImpl implements Stream {
*
* @param obj an object received at the stream.
*/
- private void processReceivedObject(JSONObject obj) {
+ private void processReceivedObject(JsonObject obj) {
// assume that tweets always have a user field
if (obj.has("user")) {
receivedObjects.offer(obj);
} else {
- getLogger().warning("Received unknown object: " + obj);
+ getLogger().log(Level.WARNING, "Received unknown object: {0}", obj);
}
}
@@ -419,7 +425,7 @@ public class StreamImpl implements Stream {
return !receivedObjects.isEmpty();
}
- public JSONObject getObject() throws InterruptedException {
+ public JsonObject getObject() throws InterruptedException {
return receivedObjects.take();
}
diff --git a/src/main/Main.java b/src/main/Main.java
index 059ade6..913646e 100644
--- a/src/main/Main.java
+++ b/src/main/Main.java
@@ -2,14 +2,14 @@ package main;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
+import io.RateLimitException;
import io.Response;
import java.io.IOException;
import java.util.Arrays;
import java.util.Scanner;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import mining.TwitterApi;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
/**
* Class for manually testing the Twitter API.
@@ -116,16 +116,20 @@ public class Main {
"Available commands:"
};
- private void searchTweets(String q) throws IOException, JSONException {
- TwitterApi.Builder req = getApi().build("search/tweets");
- req.param("q", q);
- req.param("count", "100"); // max number of tweets, cannot be higher
- req.param("lang", "en");
- Response resp = req.request();
- JsonArray statuses = resp.getResp().getAsJsonObject().get("statuses").getAsJsonArray();
- for (int i = 0; i < statuses.size(); i++) {
- JsonObject tweet = statuses.get(i).getAsJsonObject();
- System.out.println(tweet);
+ private void searchTweets(String q) throws IOException {
+ try {
+ TwitterApi.Builder req = getApi().build("search/tweets");
+ req.param("q", q);
+ req.param("count", "100"); // max number of tweets, cannot be higher
+ req.param("lang", "en");
+ Response resp = req.request();
+ JsonArray statuses = resp.getResp().getAsJsonObject().get("statuses").getAsJsonArray();
+ for (int i = 0; i < statuses.size(); i++) {
+ JsonObject tweet = statuses.get(i).getAsJsonObject();
+ System.out.println(tweet);
+ }
+ } catch (RateLimitException ex) {
+ Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
@@ -142,11 +146,7 @@ public class Main {
req.param("id", getParam(0, "numerical ID of tweet"));
break;
case searchtweets:
- try {
- searchTweets(getParam(0, "search query"));
- } catch (JSONException ex) {
- throw new IOException(ex);
- }
+ searchTweets(getParam(0, "search query"));
/* no req, will be handled by search */
break;
case hack:
@@ -186,12 +186,12 @@ public class Main {
throw new AssertionError(command.name());
}
if (req != null) {
- System.err.println("Executing: " + req.toString());
- Response result = req.request();
- if (rawOutput) {
- System.out.println(result.getResp().toString());
- } else {
+ try {
+ System.err.println("Executing: " + req.toString());
+ Response result = req.request();
System.out.println(result.getResp().toString());
+ } catch (RateLimitException ex) {
+ Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
diff --git a/src/main/TweetCounter.java b/src/main/TweetCounter.java
index 6c1397a..c630cda 100644
--- a/src/main/TweetCounter.java
+++ b/src/main/TweetCounter.java
@@ -1,12 +1,12 @@
package main;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.json.JSONException;
-import org.json.JSONObject;
import provider.ResultListener;
/**
@@ -29,14 +29,14 @@ public class TweetCounter implements ResultListener {
}
@Override
- public void tweetGenerated(JSONObject obj) {
- tweetCount++;
+ public void tweetGenerated(JsonObject obj) {
try {
- JSONObject userObj = obj.getJSONObject("user");
- String screen_name = userObj.getString("screen_name");
- users.add(screen_name);
- } catch (JSONException ex) {
- LOGGER.log(Level.WARNING, "Profile is missing data", ex);
+ JsonObject userObj = obj.getAsJsonObject("user");
+ String screen_name = userObj.get("screen_name").getAsString();
+ tweetCount++;
+ users.add(screen_name);
+ } catch (JsonParseException ex) {
+ LOGGER.log(Level.WARNING, "Profile is missing data", ex);
}
}
diff --git a/src/main/TweetShell.java b/src/main/TweetShell.java
index a86bad3..956f060 100644
--- a/src/main/TweetShell.java
+++ b/src/main/TweetShell.java
@@ -1,11 +1,13 @@
package main;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
import io.CompressableDataWriter;
import io.DataWriter;
import io.OAuthRequester;
+import io.SearchImpl;
import io.StreamImpl;
import java.io.Closeable;
-import java.io.Flushable;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
@@ -15,13 +17,11 @@ import java.util.NoSuchElementException;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
-import java.util.logging.Level;
import java.util.logging.Logger;
+import mining.Search;
import mining.Stream;
import mining.TwitterApi;
import org.apache.commons.io.IOUtils;
-import org.json.JSONException;
-import org.json.JSONObject;
import provider.CompositeResultListener;
import provider.ExceptionListener;
import provider.ResultListener;
@@ -91,20 +91,21 @@ public class TweetShell implements TwitterApi.PinSupplier {
private class StreamHandler implements ResultListener {
@Override
- public void tweetGenerated(JSONObject obj) {
+ public void tweetGenerated(JsonObject obj) {
try {
- JSONObject userObj = obj.getJSONObject("user");
+ JsonObject userObj = obj.getAsJsonObject("user");
+
System.out.println("Got tweet: "
- + userObj.getString("screen_name") + ": "
- + obj.getString("text"));
- } catch (JSONException ex) {
- getLogger().log(Level.SEVERE, "Failed to parse tweet", ex);
+ + userObj.get("screen_name").getAsString() + ": "
+ + obj.get("text").getAsString());
+ } catch (JsonParseException ex) {
+ System.err.println("failed to parse tweet " + ex);
}
}
+ }
- private Logger getLogger() {
- return Logger.getLogger(getClass().getName());
- }
+ private Logger getLogger() {
+ return Logger.getLogger(getClass().getName());
}
@Override
diff --git a/src/provider/CompositeResultListener.java b/src/provider/CompositeResultListener.java
index 5120971..7a7e4bf 100644
--- a/src/provider/CompositeResultListener.java
+++ b/src/provider/CompositeResultListener.java
@@ -1,5 +1,6 @@
package provider;
+import com.google.gson.JsonObject;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
@@ -8,7 +9,6 @@ import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;
-import org.json.JSONObject;
/**
* ResultListener that can contain multiple registrations.
@@ -46,7 +46,7 @@ public class CompositeResultListener implements ResultListener, Closeable,
}
@Override
- public void tweetGenerated(JSONObject obj) {
+ public void tweetGenerated(JsonObject obj) {
for (ResultListener rl : listeners) {
rl.tweetGenerated(obj);
}
@@ -61,6 +61,7 @@ public class CompositeResultListener implements ResultListener, Closeable,
}
}
+ @Override
public void flush() {
for (ResultListener rl : listeners) {
if (rl instanceof Flushable) {
diff --git a/src/provider/ResultListener.java b/src/provider/ResultListener.java
index 6fd5e47..a97903d 100644
--- a/src/provider/ResultListener.java
+++ b/src/provider/ResultListener.java
@@ -1,6 +1,6 @@
package provider;
-import org.json.JSONObject;
+import com.google.gson.JsonObject;
/**
* Callback when a new tweet or user is received.
@@ -12,5 +12,5 @@ public interface ResultListener {
*
* @param obj A single JSON object.
*/
- public void tweetGenerated(JSONObject obj);
+ public void tweetGenerated(JsonObject obj);
}