From ad6bb6d0d2da264656c5de0e51297997374a41d6 Mon Sep 17 00:00:00 2001 From: Maurice Laveaux Date: Thu, 22 May 2014 15:41:54 +0200 Subject: Refactored JSON to GSON's implementation. * Changed JSONException to JsonParseException. --- src/io/DataWriter.java | 42 +++++++++++++++------------- src/io/StreamImpl.java | 40 +++++++++++++++------------ src/main/Main.java | 46 +++++++++++++++---------------- src/main/TweetCounter.java | 18 ++++++------ src/main/TweetShell.java | 27 +++++++++--------- src/provider/CompositeResultListener.java | 5 ++-- src/provider/ResultListener.java | 4 +-- 7 files changed, 97 insertions(+), 85 deletions(-) diff --git a/src/io/DataWriter.java b/src/io/DataWriter.java index c1e1ce9..f3d4006 100644 --- a/src/io/DataWriter.java +++ b/src/io/DataWriter.java @@ -1,5 +1,8 @@ package io; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; import java.io.BufferedReader; import java.io.Closeable; import java.io.FileInputStream; @@ -81,7 +84,7 @@ public class DataWriter implements ResultListener, Closeable, Flushable { } @Override - public void tweetGenerated(JSONObject obj) { + public void tweetGenerated(JsonObject obj) { try { // ensure that the file is open m_tweet.open(); @@ -105,19 +108,24 @@ public class DataWriter implements ResultListener, Closeable, Flushable { try { is = store.getInputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(is)); + JsonParser jsonParser = new JsonParser(); // parse each line into a JSONObject, read the id and add it to // the set of ids. while ((line = reader.readLine()) != null) { - JSONObject obj = new JSONObject(line); - long id = obj.getLong("id"); - idSet.add(id); - lineno++; + try { + JsonObject obj = jsonParser.parse(line).getAsJsonObject(); + long id = obj.get("id").getAsLong(); + idSet.add(id); + lineno++; + } catch (JsonParseException ex) { + getLogger().log(Level.WARNING, "Tweet found without an id: {0}", ex); + } } } catch (FileNotFoundException ex) { // ignore, file will be created if necessary. - } catch (JSONException | IOException ex) { + } catch (IOException ex) { if (line != null) { - getLogger().log(Level.INFO, "Last line: " + line); + getLogger().log(Level.INFO, "Last line: {0}", line); } getLogger().log(Level.WARNING, store.getFileName() + ": error occurred in file at line " + lineno, ex); @@ -134,21 +142,17 @@ public class DataWriter implements ResultListener, Closeable, Flushable { * @param output The stream to write objects to. * @param idSet The id set to add the obj id to. */ - private void writeObject(JSONObject obj, OutputStream output, + private void writeObject(JsonObject obj, OutputStream output, Set idSet) { - try { - long id = obj.getLong("id"); + long id = obj.get("id").getAsLong(); - if (!idSet.contains(id)) { - try { - output.write((obj.toString() + "\n").getBytes(Charsets.UTF_8)); - idSet.add(id); - } catch (IOException ex) { - getLogger().log(Level.WARNING, "Cannot write to file", ex); - } + if (!idSet.contains(id)) { + try { + output.write((obj.toString() + "\n").getBytes(Charsets.UTF_8)); + idSet.add(id); + } catch (IOException ex) { + getLogger().log(Level.WARNING, "Cannot write to file", ex); } - } catch (JSONException ex) { - getLogger().log(Level.WARNING, "ID not found?!", ex); } } diff --git a/src/io/StreamImpl.java b/src/io/StreamImpl.java index a222bd8..b2f6b0c 100644 --- a/src/io/StreamImpl.java +++ b/src/io/StreamImpl.java @@ -1,5 +1,9 @@ package io; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -13,15 +17,13 @@ import java.util.List; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Level; import java.util.logging.Logger; import mining.Stream; import oauth.signpost.exception.OAuthException; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; import provider.ExceptionListener; import provider.ResultListener; import support.StreamingGZIPInputStream; @@ -236,7 +238,7 @@ public class StreamImpl implements Stream { // if the worker has any old ones left. while (worker.isRunning() || worker.hasObjects()) { try { - JSONObject obj = worker.getObject(); + JsonObject obj = worker.getObject(); if (obj == worker.endOfStream) { // end of objects marker assert !worker.isRunning(); @@ -249,7 +251,7 @@ public class StreamImpl implements Stream { } } - private void processObject(JSONObject obj) { + private void processObject(JsonObject obj) { synchronized (resultListenerSync) { if (resultListener != null) { resultListener.tweetGenerated(obj); @@ -263,9 +265,9 @@ public class StreamImpl implements Stream { private final String keywords; private final HttpURLConnection connection; private volatile boolean running = true; - private final BlockingQueue receivedObjects; + private final BlockingQueue receivedObjects; private InputStream inputStream; - private final JSONObject endOfStream; + private final JsonObject endOfStream; Worker(String keywords) throws IOException { this.keywords = keywords; @@ -280,7 +282,7 @@ public class StreamImpl implements Stream { throw ex; } this.receivedObjects = new LinkedBlockingQueue<>(); - this.endOfStream = new JSONObject(); + this.endOfStream = new JsonObject(); } /** @@ -382,17 +384,21 @@ public class StreamImpl implements Stream { // See https://dev.twitter.com/docs/streaming-apis/messages InputStreamReader isr = new InputStreamReader(is, Charsets.UTF_8); BufferedReader reader = new BufferedReader(isr); - JSONTokener jsonTokener = new JSONTokener(reader); + JsonParser parser = new JsonParser(); while (running) { try { - Object obj = jsonTokener.nextValue(); - if (obj instanceof JSONObject) { - processReceivedObject((JSONObject) obj); + String line = reader.readLine(); + if (line.isEmpty()) { + continue; + } + JsonElement elem = parser.parse(line); + if (elem.isJsonObject()) { + processReceivedObject(elem.getAsJsonObject()); } else { - getLogger().severe("Got unexpected object: " + obj); + getLogger().log(Level.SEVERE, "Got unexpected object: {0}", elem); throw new IOException("Got unexpected type from stream"); } - } catch (JSONException ex) { + } catch (JsonParseException ex) { // ignore IO errors for a stop request ("Socket closed") if (running) { throw new IOException(ex); @@ -406,12 +412,12 @@ public class StreamImpl implements Stream { * * @param obj an object received at the stream. */ - private void processReceivedObject(JSONObject obj) { + private void processReceivedObject(JsonObject obj) { // assume that tweets always have a user field if (obj.has("user")) { receivedObjects.offer(obj); } else { - getLogger().warning("Received unknown object: " + obj); + getLogger().log(Level.WARNING, "Received unknown object: {0}", obj); } } @@ -419,7 +425,7 @@ public class StreamImpl implements Stream { return !receivedObjects.isEmpty(); } - public JSONObject getObject() throws InterruptedException { + public JsonObject getObject() throws InterruptedException { return receivedObjects.take(); } diff --git a/src/main/Main.java b/src/main/Main.java index 059ade6..913646e 100644 --- a/src/main/Main.java +++ b/src/main/Main.java @@ -2,14 +2,14 @@ package main; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import io.RateLimitException; import io.Response; import java.io.IOException; import java.util.Arrays; import java.util.Scanner; +import java.util.logging.Level; +import java.util.logging.Logger; import mining.TwitterApi; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** * Class for manually testing the Twitter API. @@ -116,16 +116,20 @@ public class Main { "Available commands:" }; - private void searchTweets(String q) throws IOException, JSONException { - TwitterApi.Builder req = getApi().build("search/tweets"); - req.param("q", q); - req.param("count", "100"); // max number of tweets, cannot be higher - req.param("lang", "en"); - Response resp = req.request(); - JsonArray statuses = resp.getResp().getAsJsonObject().get("statuses").getAsJsonArray(); - for (int i = 0; i < statuses.size(); i++) { - JsonObject tweet = statuses.get(i).getAsJsonObject(); - System.out.println(tweet); + private void searchTweets(String q) throws IOException { + try { + TwitterApi.Builder req = getApi().build("search/tweets"); + req.param("q", q); + req.param("count", "100"); // max number of tweets, cannot be higher + req.param("lang", "en"); + Response resp = req.request(); + JsonArray statuses = resp.getResp().getAsJsonObject().get("statuses").getAsJsonArray(); + for (int i = 0; i < statuses.size(); i++) { + JsonObject tweet = statuses.get(i).getAsJsonObject(); + System.out.println(tweet); + } + } catch (RateLimitException ex) { + Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); } } @@ -142,11 +146,7 @@ public class Main { req.param("id", getParam(0, "numerical ID of tweet")); break; case searchtweets: - try { - searchTweets(getParam(0, "search query")); - } catch (JSONException ex) { - throw new IOException(ex); - } + searchTweets(getParam(0, "search query")); /* no req, will be handled by search */ break; case hack: @@ -186,12 +186,12 @@ public class Main { throw new AssertionError(command.name()); } if (req != null) { - System.err.println("Executing: " + req.toString()); - Response result = req.request(); - if (rawOutput) { - System.out.println(result.getResp().toString()); - } else { + try { + System.err.println("Executing: " + req.toString()); + Response result = req.request(); System.out.println(result.getResp().toString()); + } catch (RateLimitException ex) { + Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); } } } diff --git a/src/main/TweetCounter.java b/src/main/TweetCounter.java index 6c1397a..c630cda 100644 --- a/src/main/TweetCounter.java +++ b/src/main/TweetCounter.java @@ -1,12 +1,12 @@ package main; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; import java.util.Date; import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import org.json.JSONException; -import org.json.JSONObject; import provider.ResultListener; /** @@ -29,14 +29,14 @@ public class TweetCounter implements ResultListener { } @Override - public void tweetGenerated(JSONObject obj) { - tweetCount++; + public void tweetGenerated(JsonObject obj) { try { - JSONObject userObj = obj.getJSONObject("user"); - String screen_name = userObj.getString("screen_name"); - users.add(screen_name); - } catch (JSONException ex) { - LOGGER.log(Level.WARNING, "Profile is missing data", ex); + JsonObject userObj = obj.getAsJsonObject("user"); + String screen_name = userObj.get("screen_name").getAsString(); + tweetCount++; + users.add(screen_name); + } catch (JsonParseException ex) { + LOGGER.log(Level.WARNING, "Profile is missing data", ex); } } diff --git a/src/main/TweetShell.java b/src/main/TweetShell.java index a86bad3..956f060 100644 --- a/src/main/TweetShell.java +++ b/src/main/TweetShell.java @@ -1,11 +1,13 @@ package main; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; import io.CompressableDataWriter; import io.DataWriter; import io.OAuthRequester; +import io.SearchImpl; import io.StreamImpl; import java.io.Closeable; -import java.io.Flushable; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -15,13 +17,11 @@ import java.util.NoSuchElementException; import java.util.Scanner; import java.util.Set; import java.util.TreeMap; -import java.util.logging.Level; import java.util.logging.Logger; +import mining.Search; import mining.Stream; import mining.TwitterApi; import org.apache.commons.io.IOUtils; -import org.json.JSONException; -import org.json.JSONObject; import provider.CompositeResultListener; import provider.ExceptionListener; import provider.ResultListener; @@ -91,20 +91,21 @@ public class TweetShell implements TwitterApi.PinSupplier { private class StreamHandler implements ResultListener { @Override - public void tweetGenerated(JSONObject obj) { + public void tweetGenerated(JsonObject obj) { try { - JSONObject userObj = obj.getJSONObject("user"); + JsonObject userObj = obj.getAsJsonObject("user"); + System.out.println("Got tweet: " - + userObj.getString("screen_name") + ": " - + obj.getString("text")); - } catch (JSONException ex) { - getLogger().log(Level.SEVERE, "Failed to parse tweet", ex); + + userObj.get("screen_name").getAsString() + ": " + + obj.get("text").getAsString()); + } catch (JsonParseException ex) { + System.err.println("failed to parse tweet " + ex); } } + } - private Logger getLogger() { - return Logger.getLogger(getClass().getName()); - } + private Logger getLogger() { + return Logger.getLogger(getClass().getName()); } @Override diff --git a/src/provider/CompositeResultListener.java b/src/provider/CompositeResultListener.java index 5120971..7a7e4bf 100644 --- a/src/provider/CompositeResultListener.java +++ b/src/provider/CompositeResultListener.java @@ -1,5 +1,6 @@ package provider; +import com.google.gson.JsonObject; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; @@ -8,7 +9,6 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; -import org.json.JSONObject; /** * ResultListener that can contain multiple registrations. @@ -46,7 +46,7 @@ public class CompositeResultListener implements ResultListener, Closeable, } @Override - public void tweetGenerated(JSONObject obj) { + public void tweetGenerated(JsonObject obj) { for (ResultListener rl : listeners) { rl.tweetGenerated(obj); } @@ -61,6 +61,7 @@ public class CompositeResultListener implements ResultListener, Closeable, } } + @Override public void flush() { for (ResultListener rl : listeners) { if (rl instanceof Flushable) { diff --git a/src/provider/ResultListener.java b/src/provider/ResultListener.java index 6fd5e47..a97903d 100644 --- a/src/provider/ResultListener.java +++ b/src/provider/ResultListener.java @@ -1,6 +1,6 @@ package provider; -import org.json.JSONObject; +import com.google.gson.JsonObject; /** * Callback when a new tweet or user is received. @@ -12,5 +12,5 @@ public interface ResultListener { * * @param obj A single JSON object. */ - public void tweetGenerated(JSONObject obj); + public void tweetGenerated(JsonObject obj); } -- cgit v1.2.1