diff options
author | Peter Wu <peter@lekensteyn.nl> | 2014-05-11 13:54:59 +0200 |
---|---|---|
committer | Peter Wu <peter@lekensteyn.nl> | 2014-05-11 13:54:59 +0200 |
commit | dfe27ab80266b7191e36cbc95f3cd9dacafb6c5f (patch) | |
tree | 51b8f58e31f021bbb0150954f9429668bb555c8d /src/database/QueryUtils.java | |
parent | 16d71f8894963b7495ca0e57d50b91142ed3b506 (diff) | |
download | Datafiller-dfe27ab80266b7191e36cbc95f3cd9dacafb6c5f.tar.gz |
More data sanization (decode HTML entities, location fix)
Diffstat (limited to 'src/database/QueryUtils.java')
-rw-r--r-- | src/database/QueryUtils.java | 16 |
1 files changed, 13 insertions, 3 deletions
diff --git a/src/database/QueryUtils.java b/src/database/QueryUtils.java index 678df8c..b103a06 100644 --- a/src/database/QueryUtils.java +++ b/src/database/QueryUtils.java @@ -93,12 +93,12 @@ public class QueryUtils { public static void setInsertParams(NamedPreparedStatement tweetStatement, NamedPreparedStatement profileStatement, - Tweet tweet) throws SQLException { + Tweet tweet, String tweetText) throws SQLException { tweetStatement.setLong("tweetid", tweet.id); tweetStatement.setTimestamp("createdat", tweet.created_at); tweetStatement.setInt("favcount", tweet.favorite_count); tweetStatement.setLong("retweetcount", tweet.retweet_count); - tweetStatement.setString("text", tweet.text); + tweetStatement.setString("text", tweetText); if (tweet.coordinates != null) { float[] coords = tweet.coordinates.coordinates; String coords_str = String.format("%f,%f", coords[0], coords[1]); @@ -128,7 +128,8 @@ public class QueryUtils { profileStatement.setInt("tweetcount", twuser.statuses_count); profileStatement.setInt("followercount", twuser.followers_count); profileStatement.setInt("followedcount", twuser.friends_count); - profileStatement.setString("location", twuser.location); + String userLocation = getUserLocation(twuser); + profileStatement.setString("location", userLocation); profileStatement.setString("tweetname", twuser.screen_name); profileStatement.setTimestamp("createdat", twuser.created_at); profileStatement.setString("language", twuser.lang); @@ -142,4 +143,13 @@ public class QueryUtils { brandStmt.setString("brand", brand); // TODO: rating (positive) } + + private static String getUserLocation(User user) { + String location = user.location; + if (location != null && location.contains("\0")) { + System.err.println("Warning: \\0 location found for user " + user); + location = location.replace("\0", ""); + } + return location; + } } |