summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Wu <peter@lekensteyn.nl>2014-05-09 23:03:09 +0200
committerPeter Wu <peter@lekensteyn.nl>2014-05-09 23:03:09 +0200
commit34c600891613280e41cfd1ec1ad1bee8f23d043a (patch)
tree5d08b0673ff939d5bb066b4ac488c7623a86b1cf
parent9be617801a8dabe4bcae017c2f671c5c10b3aebb (diff)
downloadDatafiller-34c600891613280e41cfd1ec1ad1bee8f23d043a.tar.gz
Entities can be missing, user.place is not a string
* User: place is not a string but a Place object. * User: entities is nullable. * Tweet: in_reply_to_user_id, coordinates is nullable. * ValidatingJsonDeserializer: Treat null values as missing fields. * ValidatingJsonDeserializerTest: Test for null values.
-rw-r--r--src/data/Tweet.java16
-rw-r--r--src/data/User.java1
-rw-r--r--src/data/ValidatingJsonDeserializer.java5
-rw-r--r--src/database/QueryUtils.java6
-rw-r--r--test/data/ValidatingJsonDeserializerTest.java52
5 files changed, 71 insertions, 9 deletions
diff --git a/src/data/Tweet.java b/src/data/Tweet.java
index 67292ec..384a9e1 100644
--- a/src/data/Tweet.java
+++ b/src/data/Tweet.java
@@ -10,10 +10,14 @@ public class Tweet {
public long id;
public String lang;
+ @ValidatingJsonDeserializer.Nullable
public long in_reply_to_user_id;
public String created_at;
public long favorite_count;
- public String place;
+ @ValidatingJsonDeserializer.Nullable
+ @ValidatingJsonDeserializer.Validator
+ public Place place;
+ @ValidatingJsonDeserializer.Nullable
public String coordinates;
public String text;
@ValidatingJsonDeserializer.Nullable
@@ -31,6 +35,16 @@ public class Tweet {
return gson.toJson(this);
}
+ public static class Place {
+
+ //public String id; // "a5b6bdd8008412b1"
+ //public String name; // "Danbury"
+ //public String country_code; // "US"
+ public String country; // "United States"
+ //public String url; // "https://api.twitter.com/1.1/geo/id/a5b6bdd8008412b1.json"
+ public String full_name; // "Danbury, CT"
+ }
+
public static class Entities {
@ValidatingJsonDeserializer.Validator
diff --git a/src/data/User.java b/src/data/User.java
index 390d986..97061bc 100644
--- a/src/data/User.java
+++ b/src/data/User.java
@@ -14,6 +14,7 @@ public class User {
public String location;
public String screen_name;
public String created_at;
+ @ValidatingJsonDeserializer.Nullable
@ValidatingJsonDeserializer.Validator
public Entities entities;
public String lang;
diff --git a/src/data/ValidatingJsonDeserializer.java b/src/data/ValidatingJsonDeserializer.java
index 511264d..c168821 100644
--- a/src/data/ValidatingJsonDeserializer.java
+++ b/src/data/ValidatingJsonDeserializer.java
@@ -33,14 +33,15 @@ public class ValidatingJsonDeserializer<T> implements JsonDeserializer<T> {
private void checkObject(String path, JsonElement je, Class type) {
JsonObject jsonObj = je.getAsJsonObject();
for (Field f : type.getDeclaredFields()) {
- if (!jsonObj.has(f.getName())) {
+ JsonElement val = jsonObj.get(f.getName());
+ if (!jsonObj.has(f.getName()) || val.isJsonNull()) {
if (f.getAnnotation(Nullable.class) != null) {
// null allowed, skip
continue;
}
throw new JsonParseException("Missing field: " + path + f.getName());
}
- tryValidateProperty(path, jsonObj.get(f.getName()), f);
+ tryValidateProperty(path, val, f);
// TODO: validate type?
}
}
diff --git a/src/database/QueryUtils.java b/src/database/QueryUtils.java
index f87ba47..228ccc8 100644
--- a/src/database/QueryUtils.java
+++ b/src/database/QueryUtils.java
@@ -103,7 +103,11 @@ public class QueryUtils {
}
tweetStatement.setLong("replyid", tweet.in_reply_to_user_id);
// TODO: place is not a string...
- tweetStatement.setString("place", tweet.place);
+ if (tweet.place != null) {
+ tweetStatement.setString("place", tweet.place.full_name + " " + tweet.place.country);
+ } else {
+ tweetStatement.setString("place", null);
+ }
User twuser = tweet.user;
tweetStatement.setLong("userid", twuser.id);
diff --git a/test/data/ValidatingJsonDeserializerTest.java b/test/data/ValidatingJsonDeserializerTest.java
index 84dde5c..e258be2 100644
--- a/test/data/ValidatingJsonDeserializerTest.java
+++ b/test/data/ValidatingJsonDeserializerTest.java
@@ -3,6 +3,7 @@ package data;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
@@ -359,7 +360,7 @@ public class ValidatingJsonDeserializerTest {
tweet.addProperty("in_reply_to_user_id", 4);
tweet.addProperty("created_at", "X");
tweet.addProperty("favorite_count", 4);
- tweet.addProperty("place", "X");
+ tweet.add("place", JsonNull.INSTANCE);
tweet.addProperty("coordinates", "X");
tweet.addProperty("text", "X");
tweet.add("retweeted_status", JsonNull.INSTANCE); // Tweet object
@@ -385,6 +386,19 @@ public class ValidatingJsonDeserializerTest {
obj.remove(prop);
}
+ private void addProperty(JsonObject obj, JsonElement val, String... names) {
+ String prop;
+ for (int i = 0; i < names.length - 1; i++) {
+ prop = names[i];
+ if (!obj.has(prop)) {
+ obj.add(prop, new JsonObject());
+ }
+ obj = obj.getAsJsonObject(prop);
+ }
+ prop = names[names.length - 1];
+ obj.add(prop, val);
+ }
+
private void checkImpairedTweet(String... names) {
JsonObject tweet = buildMinimalTweet(buildMinimalUser());
removeProperty(tweet, names);
@@ -430,17 +444,30 @@ public class ValidatingJsonDeserializerTest {
public void testTweet() {
checkImpairedTweet("id");
checkImpairedTweet("lang");
- checkImpairedTweet("in_reply_to_user_id");
checkImpairedTweet("created_at");
checkImpairedTweet("favorite_count");
- checkImpairedTweet("place");
- checkImpairedTweet("coordinates");
checkImpairedTweet("text");
checkImpairedTweet("retweet_count");
checkImpairedTweet("user");
}
@Test
+ public void testTweetNulls() {
+ // place can be null
+ JsonObject tweet = buildMinimalTweet(buildMinimalUser());
+ removeProperty(tweet, "place");
+ checkTweetPass(tweet);
+
+ tweet = buildMinimalTweet(buildMinimalUser());
+ removeProperty(tweet, "in_reply_to_user_id");
+ checkTweetPass(tweet);
+
+ tweet = buildMinimalTweet(buildMinimalUser());
+ removeProperty(tweet, "coordinates");
+ checkTweetPass(tweet);
+ }
+
+ @Test
public void testTweetEntities() {
checkImpairedTweet("entities");
checkImpairedTweet("entities", "hashtags");
@@ -462,8 +489,15 @@ public class ValidatingJsonDeserializerTest {
}
@Test
+ public void testNullUserEntities() {
+ // entities can be null
+ JsonObject tweet = buildMinimalTweet(buildMinimalUser());
+ removeProperty(tweet, "user", "entities");
+ checkTweetPass(tweet);
+ }
+
+ @Test
public void testUserEntities() {
- checkImpairedTweet("user", "entities");
checkImpairedTweet("user", "entities", "url");
checkImpairedTweet("user", "entities", "url", "urls");
}
@@ -489,4 +523,12 @@ public class ValidatingJsonDeserializerTest {
tweet.add("retweeted_status", retweet);
checkTweetFail(tweet, "Missing field: retweeted_status.text");
}
+
+ @Test
+ public void testNullPlace() {
+ JsonObject tweet = buildMinimalTweet(buildMinimalUser());
+ removeProperty(tweet, "place");
+ tweet.add("place", null);
+ checkTweetPass(tweet);
+ }
}