From d05e3c54262d196d43955701bc555d096607ddb0 Mon Sep 17 00:00:00 2001 From: Maurice Laveaux Date: Wed, 30 Apr 2014 11:55:33 +0200 Subject: Removed unused data classes * data will be stored as JSONObjects into files. --- src/data/Profile.java | 16 ---------------- src/data/Tweet.java | 9 --------- 2 files changed, 25 deletions(-) delete mode 100644 src/data/Profile.java delete mode 100644 src/data/Tweet.java diff --git a/src/data/Profile.java b/src/data/Profile.java deleted file mode 100644 index 4e1f413..0000000 --- a/src/data/Profile.java +++ /dev/null @@ -1,16 +0,0 @@ -package data; - -/** - * This class contains all data that is stored of a user by twitter. - */ -public class Profile { - - // The displayed name of the user. - private String m_displayName; - - // The real name of the user (when set) - private String m_realName; - - // When the profile is created. - private String m_creationDate; -} diff --git a/src/data/Tweet.java b/src/data/Tweet.java deleted file mode 100644 index ebaabf3..0000000 --- a/src/data/Tweet.java +++ /dev/null @@ -1,9 +0,0 @@ - -package data; - -/** - * This class contains the data that is stored within a Tweet. - */ -public class Tweet { - -} -- cgit v1.2.1 From e9eab0534b4cb647bfe705d4313d4fe5a1d4a8fe Mon Sep 17 00:00:00 2001 From: Maurice Laveaux Date: Wed, 30 Apr 2014 11:56:55 +0200 Subject: Added comments to the Listener interfaces. --- src/provider/ProfileListener.java | 5 +++++ src/provider/TweetListener.java | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/provider/ProfileListener.java b/src/provider/ProfileListener.java index 23304c9..4232001 100644 --- a/src/provider/ProfileListener.java +++ b/src/provider/ProfileListener.java @@ -7,5 +7,10 @@ import org.json.JSONObject; */ public interface ProfileListener { + /** + * This method is called when a new profile is provided. + * + * @param obj A single JSON object + */ void profileGenerated(JSONObject obj); } diff --git a/src/provider/TweetListener.java b/src/provider/TweetListener.java index e58d580..5b1df0e 100644 --- a/src/provider/TweetListener.java +++ b/src/provider/TweetListener.java @@ -1,4 +1,4 @@ -package provider; + package provider; import org.json.JSONObject; @@ -7,5 +7,10 @@ import org.json.JSONObject; */ public interface TweetListener { + /** + * This method is called when a new tweet is provided. + * + * @param obj A single JSON object. + */ public void tweetGenerated(JSONObject obj); } -- cgit v1.2.1 From 375458f918bd53ae08b1dd0234ec09f85e056930 Mon Sep 17 00:00:00 2001 From: Maurice Laveaux Date: Wed, 30 Apr 2014 12:18:48 +0200 Subject: Removed the unused database.txt, added profiles.txt and tweets.txt to the .gitignore --- .gitignore | 3 ++- database.txt | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 database.txt diff --git a/.gitignore b/.gitignore index 01c0371..212e315 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ jacoco.exec-* .*.sw? *.orig config.txt -database.txt +tweets.txt +profiles.txt diff --git a/database.txt b/database.txt deleted file mode 100644 index 586afa9..0000000 --- a/database.txt +++ /dev/null @@ -1,2 +0,0 @@ - - {"location":"","default_profile":true,"profile_background_tile":false,"statuses_count":4,"lang":"nl","profile_link_color":"0084B4","id":745405160,"following":null,"protected":false,"favourites_count":1,"profile_text_color":"333333","description":"Stuff","verified":false,"contributors_enabled":false,"profile_sidebar_border_color":"C0DEED","name":"MazK","profile_background_color":"C0DEED","created_at":"Wed Aug 08 15:27:55 +0000 2012","is_translation_enabled":false,"default_profile_image":false,"followers_count":1,"profile_image_url_https":"https://pbs.twimg.com/profile_images/2482341510/o83ce5xo4s54ugq7xbl9_normal.gif","geo_enabled":false,"status":{"contributors":null,"text":"@planetside2 I like the \"Time Lapse Weapon Design\" the most, because it's nice to get an impression of how this game was designed.","geo":null,"retweeted":false,"in_reply_to_screen_name":"planetside2","truncated":false,"lang":"en","entities":{"symbols":[],"urls":[],"hashtags":[],"user_mentions":[{"id":247430686,"name":"PlanetSide 2","indices":[0,12],"screen_name":"planetside2","id_str":"247430686"}]},"in_reply_to_status_id_str":"233656189089640448","id":233659630595764224,"source":"web","in_reply_to_user_id_str":"247430686","favorited":false,"in_reply_to_status_id":233656189089640448,"retweet_count":0,"created_at":"Thu Aug 09 20:22:51 +0000 2012","in_reply_to_user_id":247430686,"favorite_count":0,"id_str":"233659630595764224","place":null,"coordinates":null},"profile_background_image_url":"http://abs.twimg.com/images/themes/theme1/bg.png","profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme1/bg.png","follow_request_sent":null,"entities":{"description":{"urls":[]}},"url":null,"utc_offset":7200,"time_zone":"Amsterdam","notifications":null,"profile_use_background_image":true,"friends_count":4,"profile_sidebar_fill_color":"DDEEF6","screen_name":"maskman113","id_str":"745405160","profile_image_url":"http://pbs.twimg.com/profile_images/2482341510/o83ce5xo4s54ugq7xbl9_normal.gif","listed_count":0,"is_translator":false} \ No newline at end of file -- cgit v1.2.1 From 1527c87edbc8f5824f460c3b2c452db50f7e0640 Mon Sep 17 00:00:00 2001 From: Maurice Laveaux Date: Wed, 30 Apr 2014 12:20:15 +0200 Subject: Create a class that implements ProfileListener and TweetListener. * Creates tweets.txt and profiles.txt when they don't exist. * Read these files to store a set of ids. * Consumes new tweets and profiles and writes if the ids are not found. --- src/io/OutputStream.java | 130 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/io/OutputStream.java diff --git a/src/io/OutputStream.java b/src/io/OutputStream.java new file mode 100644 index 0000000..5a92c59 --- /dev/null +++ b/src/io/OutputStream.java @@ -0,0 +1,130 @@ +package io; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashSet; +import java.util.Scanner; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.json.JSONException; +import org.json.JSONObject; +import provider.ProfileListener; +import provider.TweetListener; + +/** + * This class writes the output data into seperate files. + * + * @author Maurice Laveaux + */ +public class OutputStream implements ProfileListener, TweetListener { + + /** + * The writer for the tweet stream. + */ + private final FileWriter m_tweetWriter; + + /** + * the writer for the profile stream. + */ + private final FileWriter m_profileWriter; + + /** + * the buffer of tweet ids that already exist. + */ + private final Set m_tweetIdSet; + + /** + * the buffer of profile ids that already exist. + */ + private final Set m_profileIdSet; + + private static final String profilesName = "profiles.txt"; + private static final String tweetsName = "tweets.txt"; + + /** + * Opens a stream to every single file that data will be streamed to. + */ + public OutputStream() { + try { + m_profileIdSet = readIds(profilesName); + m_profileWriter = new FileWriter(profilesName, true); + + m_tweetIdSet = readIds(tweetsName); + m_tweetWriter = new FileWriter(tweetsName, true); + } catch (IOException ex) { + // This should not happen. + throw new RuntimeException(ex.getMessage()); + } + } + + @Override + public void profileGenerated(JSONObject obj) { + writeObject(obj, m_profileWriter, m_profileIdSet); + } + + @Override + public void tweetGenerated(JSONObject obj) { + writeObject(obj, m_tweetWriter, m_tweetIdSet); + } + + /** + * Read the current existing tweetName and profileName filenames and fill + * the existing id set. + * + * @param filename The file to parse + * @return The set of ids + */ + private Set readIds(String filename) throws IOException { + try { + Scanner reader = new Scanner(new File(filename)); + + // TODO: Read the file JSON objects and parse the ids. + Set idSet = new HashSet(); + + // parse each line into a JSONObject, read the id and add it to + // the set of ids. + while(reader.hasNext()) { + JSONObject obj = new JSONObject(reader.nextLine()); + long id = obj.getLong("id"); + idSet.add(id); + } + + return idSet; + } catch (FileNotFoundException ex) { + // File does not exist, so create one. + File file = new File(filename); + // Return value should always be true. + file.createNewFile(); + } catch (JSONException ex) { + getLogger().log(Level.SEVERE, null, ex); + } + + // return the empty set. + return new HashSet(); + } + + private void writeObject(JSONObject obj, FileWriter writer, Set idSet) { + try { + long id = obj.getLong("id"); + + if (!idSet.contains(id)) { + // Write a single profile into the profile file. + try { + writer.write(obj.toString() + "\n"); + idSet.add(id); + } catch (IOException ex) { + getLogger().log(Level.SEVERE, null, ex); + } + } + } catch (JSONException ex) { + getLogger().log(Level.SEVERE, null, ex); + } + } + + private Logger getLogger() { + return Logger.getLogger(OutputStream.class.getName()); + } +} -- cgit v1.2.1