Initial commit

build.xml, etc. are modified a bit after opening in Netbeans 7.4.

9 files changed, 1652 insertions, 0 deletions

diff --git a/src/Chapter2/Location/LocationTranslationExample.java b/src/Chapter2/Location/LocationTranslationExample.java
new file mode 100644
index 0000000..69178dc
--- /dev/null
+++ b/src/Chapter2/Location/LocationTranslationExample.java
@@ -0,0 +1,124 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.Location;
+
+import Chapter2.support.Location;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLEncoder;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.json.JSONArray;
+import org.json.JSONException;
+
+public class LocationTranslationExample
+{
+
+      /**
+        * Translates a location string to coordinates using the database or Nominatim Service
+        * @param loc
+        * @return
+        */
+   public Location TranslateLoc(String loc)
+   {
+        if(loc!=null&&!loc.isEmpty())
+        {
+            String encodedLoc="";
+            try {
+                //Step 1: Encode the location name
+                 encodedLoc = URLEncoder.encode(loc, "UTF-8");
+            } catch (UnsupportedEncodingException ex) {
+                Logger.getLogger(LocationTranslationExample.class.getName()).log(Level.SEVERE, null, ex);
+            }
+             //Step 2: Create a get request to MapQuest API with the name of the location
+            String url= "http://open.mapquestapi.com/nominatim/v1/search?q="+encodedLoc+"&format=json";
+            String page = ReadHTML(url);
+            if(page!=null)
+            {
+                try{
+                    JSONArray results = new JSONArray(page);
+                   if(results.length()>0)
+                    {
+                       //Step 3: Read and extract the coordinates of the location as a JSONObject
+                        Location loca = new Location(results.getJSONObject(0).getDouble("lat"),results.getJSONObject(0).getDouble("lon"));
+                        return loca;
+                    }
+                }catch(JSONException ex)
+                {
+                    Logger.getLogger(LocationTranslationExample.class.getName()).log(Level.SEVERE, null, ex);
+                }
+            }
+        }
+        return null;
+  }
+
+    /**
+     * Extracts the html content of a URL
+     * @param url
+     * @return html page
+     */
+    public String ReadHTML(String url)
+    {
+        URLConnection conn = null;
+        URL theURL = null;
+        try
+        {
+                theURL = new URL(url);
+        }
+        catch ( MalformedURLException e)
+        {
+                System.out.println("Bad URL: " + theURL);
+                return null;
+        }
+        String page = "";
+        try
+        {
+            conn = theURL.openConnection();
+            HttpURLConnection huc = (HttpURLConnection) conn;
+            conn.setConnectTimeout(2000);
+            huc.setRequestProperty("User-Agent", "Mozilla/4.5");
+            //Set your email address in the request so MapQuest knows how to reach you in the event of problems
+            huc.setRequestProperty("Email", "twitterdataanalytics@gmail.com");
+            if(huc.getResponseCode()>=400&&huc.getResponseCode()<=404)
+            {
+                 return null;
+            }
+            conn.connect();
+            BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) conn.getContent()));
+            String temp=null;
+             while( (temp= bRead.readLine())!=null)
+              {
+                  page = page+"\n"+temp;
+              }
+              bRead.close();
+        }
+        catch (IOException e) {
+                //System.out.print("ReadHTML IO Error:" + e.getMessage()+" \n");
+                return null;
+        }
+	return page;
+    }
+
+    public static void main(String[] args)
+    {
+        LocationTranslationExample lte = new LocationTranslationExample();
+        if(args!=null)
+        {
+            if(args.length>0)
+            {
+                for(int i=0;i<args.length;i++)
+                {
+                    System.out.println(lte.TranslateLoc(args[i]).toString());
+                }
+            }
+        }
+    }
+}
diff --git a/src/Chapter2/openauthentication/OAuthExample.java b/src/Chapter2/openauthentication/OAuthExample.java
new file mode 100644
index 0000000..9b2ec7a
--- /dev/null
+++ b/src/Chapter2/openauthentication/OAuthExample.java
@@ -0,0 +1,79 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.openauthentication;
+
+import Chapter2.support.OAuthTokenSecret;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import oauth.signpost.OAuth;
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.OAuthProvider;
+import oauth.signpost.basic.DefaultOAuthProvider;
+import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
+import oauth.signpost.exception.OAuthCommunicationException;
+import oauth.signpost.exception.OAuthExpectationFailedException;
+import oauth.signpost.exception.OAuthMessageSignerException;
+import oauth.signpost.exception.OAuthNotAuthorizedException;
+import utils.OAuthUtils;
+
+public class OAuthExample
+{        
+    public OAuthTokenSecret GetUserAccessKeySecret()
+    {
+        try {
+            //consumer key for Twitter Data Analytics application
+            if(OAuthUtils.CONSUMER_KEY.isEmpty())
+            {
+                System.out.println("Register an application and copy the consumer key into the configuration file.");
+                return null;
+            }
+            if(OAuthUtils.CONSUMER_SECRET.isEmpty())
+            {
+                System.out.println("Register an application and copy the consumer secret into the configuration file.");
+                return null;
+            }
+            OAuthConsumer consumer = new CommonsHttpOAuthConsumer(OAuthUtils.CONSUMER_KEY,OAuthUtils.CONSUMER_SECRET);
+            OAuthProvider provider = new DefaultOAuthProvider(OAuthUtils.REQUEST_TOKEN_URL, OAuthUtils.ACCESS_TOKEN_URL, OAuthUtils.AUTHORIZE_URL);
+            String authUrl = provider.retrieveRequestToken(consumer, OAuth.OUT_OF_BAND);
+            System.out.println("Now visit:\n" + authUrl + "\n and grant this app authorization");
+            System.out.println("Enter the PIN code and hit ENTER when you're done:");
+            BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+            String pin = br.readLine();
+            System.out.println("Fetching access token from Twitter");
+            provider.retrieveAccessToken(consumer,pin);
+            String accesstoken = consumer.getToken();
+            String accesssecret  = consumer.getTokenSecret();
+            OAuthTokenSecret tokensecret = new OAuthTokenSecret(accesstoken,accesssecret);
+            return tokensecret;
+        } catch (OAuthNotAuthorizedException ex) {
+                ex.printStackTrace();
+        } catch (OAuthMessageSignerException ex) {
+                ex.printStackTrace();
+        } catch (OAuthExpectationFailedException ex) {
+                ex.printStackTrace();
+        } catch (OAuthCommunicationException ex) {
+                ex.printStackTrace();
+        } catch(IOException ex)
+        {
+            ex.printStackTrace();
+        }
+        return null;
+    }
+
+    public static OAuthTokenSecret DEBUGUserAccessSecret()
+    {
+        String accesstoken = "1262619914-tcCPB1SyXy3BMuui9OAhprcPmqg3z2csSjDSCNY";
+        String accesssecret = "cXXO0qFLBjLXGtE97pnf5Vv1RZGxZ2FZ97wCYiaVU";
+        OAuthTokenSecret tokensecret = new OAuthTokenSecret(accesstoken,accesssecret);
+        return tokensecret;
+    }
+
+    public static void main(String[] args)
+    {
+        OAuthExample aue = new OAuthExample();
+        OAuthTokenSecret tokensecret = aue.GetUserAccessKeySecret();
+        System.out.println(tokensecret.toString());
+    }
+}
diff --git a/src/Chapter2/restapi/RESTApiExample.java b/src/Chapter2/restapi/RESTApiExample.java
new file mode 100644
index 0000000..9ceb88b
--- /dev/null
+++ b/src/Chapter2/restapi/RESTApiExample.java
@@ -0,0 +1,676 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.restapi;
+
+import Chapter2.support.APIType;
+import Chapter2.support.OAuthTokenSecret;
+import Chapter2.openauthentication.OAuthExample;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.basic.DefaultOAuthConsumer;
+import oauth.signpost.exception.OAuthCommunicationException;
+import oauth.signpost.exception.OAuthExpectationFailedException;
+import oauth.signpost.exception.OAuthMessageSignerException;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class RESTApiExample
+{
+    //file handlers to store the collected user information
+    BufferedWriter OutFileWriter;
+    OAuthTokenSecret OAuthTokens;
+    /**
+     * name of the file containing a list of users
+     */
+    final String DEF_FILENAME = "users.txt";
+    final String DEF_OUTFILENAME = "restapiresults.json";
+    ArrayList<String> Usernames = new ArrayList<String>();
+    OAuthConsumer Consumer;
+
+    /**
+     * Creates a OAuthConsumer with the current consumer & user access tokens and secrets
+     * @return consumer
+     */
+    public OAuthConsumer GetConsumer()
+    {
+        OAuthConsumer consumer = new DefaultOAuthConsumer(utils.OAuthUtils.CONSUMER_KEY,utils.OAuthUtils.CONSUMER_SECRET);
+        consumer.setTokenWithSecret(OAuthTokens.getAccessToken(),OAuthTokens.getAccessSecret());
+        return consumer;
+    }
+    
+    /**
+     * Reads the file and loads the users in the file to be crawled
+     * @param filename
+     */
+    public void ReadUsers(String filename)
+    {
+        BufferedReader br = null;
+        try {
+            br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8"));
+            String temp = "";
+            while((temp = br.readLine())!=null)
+            {
+                if(!temp.isEmpty())
+                {
+                    Usernames.add(temp);
+                }
+            }
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+        finally{
+            try {
+                br.close();
+            } catch (IOException ex) {
+                ex.printStackTrace();
+            }
+        }
+    }
+
+    /**
+     * Load the User Access Token, and the User Access Secret
+     */
+    public void LoadTwitterToken()
+    {
+        //Un-comment before release
+//        OAuthExample oae = new OAuthExample();
+//        OAuthTokens =  oae.GetUserAccessKeySecret();
+        //Remove before release
+        OAuthTokens = OAuthExample.DEBUGUserAccessSecret();
+    }
+
+    public static void main(String[] args)
+    {
+        RESTApiExample rae = new RESTApiExample();
+        rae.LoadTwitterToken();
+        rae.Consumer = rae.GetConsumer();
+//        System.out.println(rae.GetStatuses("twtanalyticsbk"));
+        System.out.println(rae.GetRateLimitStatus());
+//        int apicode = InfoType.PROFILE_INFO;
+//        String infilename = rae.DEF_FILENAME;
+//        String outfilename = rae.DEF_OUTFILENAME;
+//        if(args!=null)
+//        {
+//            if(args.length>2)
+//            {
+//                apicode = Integer.parseInt(args[2]);
+//                outfilename = args[1];
+//                infilename = args[0];
+//            }
+//            if(args.length>1)
+//            {
+//                outfilename = args[1];
+//                infilename = args[0];
+//            }
+//            else
+//            if(args.length>0)
+//            {
+//                infilename = args[0];
+//            }
+//        }
+//        rae.InitializeWriters(outfilename);
+//        rae.ReadUsers(infilename);
+//        if(apicode!=InfoType.PROFILE_INFO&&apicode!=InfoType.FOLLOWER_INFO&&apicode!=InfoType.FRIEND_INFO&&apicode!=InfoType.STATUSES_INFO)
+//        {
+//          System.out.println("Invalid API type: Use 0 for Profile, 1 for Followers, 2 for Friends, and 3 for Statuses");
+//          System.exit(0);
+//        }
+//        if(rae.Usernames.size()>0)
+//        {
+//            //TO-DO: Print the possible API types and get user selection to crawl the users.
+//            rae.LoadTwitterToken();
+//            for(String user:rae.Usernames)
+//            {
+//                if(apicode==InfoType.PROFILE_INFO)
+//                {
+//                    JSONObject jobj = rae.GetProfile(user);
+//                    if(jobj!=null&&jobj.length()==0)
+//                    {
+//                        rae.WriteToFile(user, jobj.toString());
+//                    }
+//                }
+//                else
+//                if(apicode==InfoType.FRIEND_INFO)
+//                {
+//                    JSONArray statusarr = rae.GetFriends(user);
+//                    if(statusarr.length()>0)
+//                    {
+//                        rae.WriteToFile(user, statusarr.toString());
+//                    }
+//                }
+//                else
+//                if(apicode == InfoType.FOLLOWER_INFO)
+//                {
+//                    JSONArray statusarr = rae.GetFollowers(user);
+//                    if(statusarr.length()>0)
+//                    {
+//                        rae.WriteToFile(user, statusarr.toString());
+//                    }
+//                }
+//                else
+//                if(apicode == InfoType.STATUSES_INFO)
+//                {
+//                    JSONArray statusarr = rae.GetStatuses(user);
+//                    if(statusarr.length()>0)
+//                    {
+//                        rae.GetStatuses(user);
+//                    }
+//                }
+//            }
+//        }
+////        now you can close the files as all the threads have finished
+//        rae.CleanupAfterFinish();
+   }
+
+    /**
+     * Retrieves the rate limit status of the application
+     * @return
+     */
+   public JSONObject GetRateLimitStatus()
+   {
+     try{
+            URL url = new URL("https://api.twitter.com/1.1/application/rate_limit_status.json");
+            HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+            huc.setReadTimeout(5000);           
+            Consumer.sign(huc);
+            huc.connect();
+            BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
+            StringBuffer page = new StringBuffer();
+            String temp= "";
+            while((temp = bRead.readLine())!=null)
+            {
+                page.append(temp);
+            }
+            bRead.close();
+            return (new JSONObject(page.toString()));
+        } catch (JSONException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthCommunicationException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }  catch (OAuthMessageSignerException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthExpectationFailedException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }catch(IOException ex)
+        {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }
+     return null;
+   }
+
+   /**
+    * Initialize the file writer
+    * @param path of the file
+    * @param outFilename name of the file
+    */
+   public void InitializeWriters(String outFilename) {
+        try {
+            File fl = new File(outFilename);
+            if(!fl.exists())
+            {
+                fl.createNewFile();
+            }
+            /**
+             * Use UTF-8 encoding when saving files to avoid
+             * losing Unicode characters in the data
+             */
+            OutFileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilename,true),"UTF-8"));
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+   /**
+    * Close the opened filewriter to save the data
+    */
+   public void CleanupAfterFinish()
+   {
+        try {
+            OutFileWriter.close();
+        } catch (IOException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }
+   }
+
+   /**
+    * Writes the retrieved data to the output file
+    * @param data containing the retrived information in JSON
+    * @param user name of the user currently being written
+    */
+    public void WriteToFile(String user, String data)
+    {
+        try
+        {
+            OutFileWriter.write(data);
+            OutFileWriter.newLine();
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    /**
+     * Retrives the profile information of the user
+     * @param username of the user whose profile needs to be retrieved
+     * @return the profile information as a JSONObject
+     */
+    public JSONObject GetProfile(String username)
+    {
+            BufferedReader bRead = null;
+            JSONObject profile = null;
+            try {
+            System.out.println("Processing profile of "+username);
+            boolean flag = true;
+            URL url = new URL("https://api.twitter.com/1.1/users/show.json?screen_name="+username);
+            HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+            huc.setReadTimeout(5000);
+            // Step 2: Sign the request using the OAuth Secret
+            Consumer.sign(huc);
+            huc.connect();
+            if(huc.getResponseCode()==404||huc.getResponseCode()==401)
+            {
+               System.out.println(huc.getResponseMessage());
+            }           
+            else
+            if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
+            {
+                try {
+                    huc.disconnect();
+                    System.out.println(huc.getResponseMessage());
+                    Thread.sleep(3000);
+                } catch (InterruptedException ex) {
+                    ex.printStackTrace();
+                }
+            }
+            else
+                // Step 3: If the requests have been exhausted, then wait until the quota is renewed
+            if(huc.getResponseCode()==429)
+            {
+                try {
+                    huc.disconnect();
+                    Thread.sleep(this.GetWaitTime("/users/show/:id"));
+                    flag = false;
+                } catch (InterruptedException ex) {
+                    ex.printStackTrace();
+                }
+            }
+            if(!flag)
+            {
+                //recreate the connection because something went wrong the first time.
+                huc.connect();
+            }
+            StringBuilder content=new StringBuilder();
+            if(flag)
+            {
+                bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
+                String temp= "";
+                while((temp = bRead.readLine())!=null)
+                {
+                    content.append(temp);
+                }
+            }
+            huc.disconnect();
+            try {
+                profile = new JSONObject(content.toString());
+            } catch (JSONException ex) {
+                ex.printStackTrace();
+            }
+        } catch (OAuthCommunicationException ex) {
+            ex.printStackTrace();
+        } catch (OAuthMessageSignerException ex) {
+            ex.printStackTrace();
+        } catch (OAuthExpectationFailedException ex) {
+            ex.printStackTrace();
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+        return profile;
+    }
+
+    /**
+     * Retrieves the followers of a user
+     * @param username the name of the user whose followers need to be retrieved
+     * @return a list of user objects corresponding to the followers of the user
+     */
+    public JSONArray GetFollowers(String username)
+    {
+        BufferedReader bRead = null;
+        JSONArray followers = new JSONArray();
+        try {
+            System.out.println(" followers user = "+username);
+            long cursor = -1;
+            while(true)
+            {
+                if(cursor==0)
+                {
+                    break;
+                }
+                // Step 1: Create the APi request using the supplied username
+                URL url = new URL("https://api.twitter.com/1.1/followers/list.json?screen_name="+username+"&cursor=" + cursor);
+                HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+                huc.setReadTimeout(5000);
+                // Step 2: Sign the request using the OAuth Secret
+                Consumer.sign(huc);
+                huc.connect();
+                if(huc.getResponseCode()==400||huc.getResponseCode()==404)
+                {
+                    System.out.println(huc.getResponseMessage());
+                    break;
+                }
+                else
+                if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503||huc.getResponseCode()==504)
+                {
+                    try{
+                        System.out.println(huc.getResponseMessage());
+                        huc.disconnect();
+                        Thread.sleep(3000);
+                        continue;
+                    } catch (InterruptedException ex) {
+                        Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+                    }
+                }
+                else
+                    // Step 3: If the requests have been exhausted, then wait until the quota is renewed
+                if(huc.getResponseCode()==429)
+                {
+                    try {
+                        huc.disconnect();
+                        Thread.sleep(this.GetWaitTime("/followers/list"));
+                        continue;
+                    } catch (InterruptedException ex) {
+                        Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+                    }
+                }
+                // Step 4: Retrieve the followers list from Twitter
+                bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
+                StringBuilder content = new StringBuilder();
+                String temp = "";
+                while((temp = bRead.readLine())!=null)
+                {
+                    content.append(temp);
+                }
+                try {
+                    JSONObject jobj = new JSONObject(content.toString());
+                    // Step 5: Retrieve the token for the next request
+                    cursor = jobj.getLong("next_cursor");
+                    JSONArray idlist = jobj.getJSONArray("users");
+                    if(idlist.length()==0)
+                    {
+                        break;
+                    }
+                    for(int i=0;i<idlist.length();i++)
+                    {
+                        followers.put(idlist.getJSONObject(i));
+                    }
+                } catch (JSONException ex) {
+                    Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+                }
+            }
+        } catch (OAuthCommunicationException ex) {
+            ex.printStackTrace();
+        } catch (OAuthMessageSignerException ex) {
+            ex.printStackTrace();
+        } catch (OAuthExpectationFailedException ex) {
+            ex.printStackTrace();
+        } catch (IOException ex) {              
+            ex.printStackTrace();
+        }
+        return followers;
+    }
+
+    /**
+     * Retrieved the status messages of a user
+     * @param username the name of the user whose status messages need to be retrieved
+     * @return a list of status messages
+     */
+    public JSONArray GetStatuses(String username)
+        {
+            BufferedReader bRead = null;
+            //Get the maximum number of tweets possible in a single page 200
+            int tweetcount = 200;
+            //Include include_rts because it is counted towards the limit anyway.
+            boolean include_rts = true;
+            JSONArray statuses = new JSONArray();
+            try {
+                System.out.println("Processing status messages of "+username);
+                long maxid = 0;
+                while(true)
+                {
+                    URL url = null;                    
+                    if(maxid==0)
+                    {
+                        url = new URL("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=" + username+"&include_rts="+include_rts+"&count="+tweetcount);
+                    }
+                    else
+                    {
+                        //use max_id to get the tweets in the next page. Use max_id-1 to avoid getting redundant tweets.
+                        url = new URL("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=" + username+"&include_rts="+include_rts+"&count="+tweetcount+"&max_id="+(maxid-1));
+                    }
+                    HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+                    huc.setReadTimeout(5000);
+                    Consumer.sign(huc);
+                    huc.connect();
+                    if(huc.getResponseCode()==400||huc.getResponseCode()==404)
+                    {
+                        System.out.println(huc.getResponseCode());
+                        break;
+                    }
+                    else
+                    if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
+                    {
+                        try {System.out.println(huc.getResponseCode());
+                            Thread.sleep(3000);
+                        } catch (InterruptedException ex) {
+                            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+                        }
+                    }
+                    else
+                        // Step 3: If the requests have been exhausted, then wait until the quota is renewed
+                    if(huc.getResponseCode()==429)
+                    {
+                        try {
+                            huc.disconnect();
+                            Thread.sleep(this.GetWaitTime("/statuses/user_timeline"));
+                            continue;
+                        } catch (InterruptedException ex) {
+                            ex.printStackTrace();
+                        }
+                    }
+                    bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getInputStream()));
+                    StringBuilder content = new StringBuilder();
+                    String temp = "";
+                    while((temp = bRead.readLine())!=null)
+                    {
+                        content.append(temp);
+                    }
+                    try {
+                        JSONArray statusarr = new JSONArray(content.toString());
+                        if(statusarr.length()==0)
+                        {
+                            break;
+                        }
+                        for(int i=0;i<statusarr.length();i++)
+                        {
+                            JSONObject jobj = statusarr.getJSONObject(i);
+                            statuses.put(jobj);
+                            //Get the max_id to get the next batch of tweets
+                            if(!jobj.isNull("id"))
+                            {
+                                maxid = jobj.getLong("id");
+                            }
+                        }
+                    } catch (JSONException ex) {
+                        ex.printStackTrace();
+                    }
+                }
+                System.out.println(statuses.length());
+        } catch (OAuthCommunicationException ex) {
+            ex.printStackTrace();
+        } catch (OAuthMessageSignerException ex) {
+            ex.printStackTrace();
+        } catch (OAuthExpectationFailedException ex) {
+            ex.printStackTrace();
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+        return statuses;
+    }
+
+    /**
+     * Retrieves the friends of a user
+     * @param username the name of the user whose friends need to be fetched
+     * @return a list of user objects who are friends of the user
+     */
+    public JSONArray GetFriends(String username)
+    {
+        BufferedReader bRead = null;
+        JSONArray friends = new JSONArray();
+        try {
+            System.out.println("Processing friends of "+username);
+            long cursor = -1;
+            while(true)
+            {
+                if(cursor==0)
+                {
+                    break;
+                }
+                // Step 1: Create the APi request using the supplied username
+                URL url = new URL("https://api.twitter.com/1.1/friends/list.json?screen_name="+username+"&cursor="+cursor);
+                HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+                huc.setReadTimeout(5000);
+                //Step 2: Sign the request using the OAuth Secret
+                Consumer.sign(huc);
+                huc.connect();
+                if(huc.getResponseCode()==400||huc.getResponseCode()==401)
+                {
+                    System.out.println(huc.getResponseMessage());
+                    break;
+                }
+                else
+                if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
+                {
+                    try {
+                        System.out.println(huc.getResponseMessage());
+                        Thread.sleep(3000);
+                        continue;
+                    } catch (InterruptedException ex) {
+                        ex.printStackTrace();
+                    }
+                }
+                else
+                    // Step 3: If the requests have been exhausted, then wait until the quota is renewed
+                if(huc.getResponseCode()==429)
+                {
+                    try {
+                        huc.disconnect();
+                        Thread.sleep(this.GetWaitTime("/friends/list"));
+                        continue;
+                    } catch (InterruptedException ex) {
+                        ex.printStackTrace();
+                    }
+                }
+                // Step 4: Retrieve the friends list from Twitter
+                bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
+                StringBuilder content = new StringBuilder();
+                String temp = "";
+                while((temp = bRead.readLine())!=null)
+                {
+                    content.append(temp);
+                }
+                try {
+                    JSONObject jobj = new JSONObject(content.toString());
+                    // Step 5: Retrieve the token for the next request
+                    cursor = jobj.getLong("next_cursor");
+                    JSONArray userlist = jobj.getJSONArray("users");
+                    if(userlist.length()==0)
+                    {
+                        break;
+                    }
+                    for(int i=0;i<userlist.length();i++)
+                    {
+                        friends.put(userlist.get(i));
+                    }
+                } catch (JSONException ex) {
+                    ex.printStackTrace();
+                }               
+                huc.disconnect();
+            }
+         } catch (OAuthCommunicationException ex) {
+            ex.printStackTrace();
+         } catch (OAuthMessageSignerException ex) {
+            ex.printStackTrace();
+         } catch (OAuthExpectationFailedException ex) {
+            ex.printStackTrace();
+         } catch (IOException ex) {
+            ex.printStackTrace();
+         }
+        return friends;
+    }
+
+    /**
+     * Retrieves the wait time if the API Rate Limit has been hit
+     * @param api the name of the API currently being used
+     * @return the number of milliseconds to wait before initiating a new request
+     */
+    public long GetWaitTime(String api)
+    {
+        JSONObject jobj = this.GetRateLimitStatus();
+        if(jobj!=null)
+        {
+            try {                
+                if(!jobj.isNull("resources"))
+                {
+                    JSONObject resourcesobj = jobj.getJSONObject("resources");
+                    JSONObject apilimit = null;
+                    if(api.equals(APIType.USER_TIMELINE))
+                    {
+                        JSONObject statusobj = resourcesobj.getJSONObject("statuses");
+                        apilimit = statusobj.getJSONObject(api);
+                    }
+                    else
+                    if(api.equals(APIType.FOLLOWERS))
+                    {
+                        JSONObject followersobj = resourcesobj.getJSONObject("followers");
+                        apilimit = followersobj.getJSONObject(api);
+                    }
+                    else
+                    if(api.equals(APIType.FRIENDS))
+                    {
+                        JSONObject friendsobj = resourcesobj.getJSONObject("friends");
+                        apilimit = friendsobj.getJSONObject(api);
+                    }
+                    else
+                    if(api.equals(APIType.USER_PROFILE))
+                    {
+                        JSONObject userobj = resourcesobj.getJSONObject("users");
+                        apilimit = userobj.getJSONObject(api);
+                    }
+                    int numremhits = apilimit.getInt("remaining");
+                    if(numremhits<=1)
+                    {
+                        long resettime = apilimit.getInt("reset");
+                        resettime = resettime*1000; //convert to milliseconds
+                        return resettime;
+                    }
+                }
+            } catch (JSONException ex) {
+                ex.printStackTrace();
+            }
+        }
+        return 0;
+    }
+}
diff --git a/src/Chapter2/restapi/RESTSearchExample.java b/src/Chapter2/restapi/RESTSearchExample.java
new file mode 100644
index 0000000..510661c
--- /dev/null
+++ b/src/Chapter2/restapi/RESTSearchExample.java
@@ -0,0 +1,311 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.restapi;
+
+import Chapter2.support.OAuthTokenSecret;
+import Chapter2.openauthentication.OAuthExample;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.basic.DefaultOAuthConsumer;
+import oauth.signpost.exception.OAuthCommunicationException;
+import oauth.signpost.exception.OAuthExpectationFailedException;
+import oauth.signpost.exception.OAuthMessageSignerException;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+public class RESTSearchExample
+{
+    BufferedWriter OutFileWriter;
+    OAuthTokenSecret OAuthTokens;
+    OAuthConsumer Consumer;
+    String query = "#protest";
+    String DEF_FILENAME = "searchresults.json";
+
+    /**
+     * Creates a OAuthConsumer with the current consumer & user access tokens and secrets
+     * @return consumer
+     */
+    public OAuthConsumer GetConsumer()
+    {
+        OAuthConsumer consumer = new DefaultOAuthConsumer(utils.OAuthUtils.CONSUMER_KEY,utils.OAuthUtils.CONSUMER_SECRET);
+        consumer.setTokenWithSecret(OAuthTokens.getAccessToken(), OAuthTokens.getAccessSecret());
+        return consumer;
+    }
+
+    /**
+     * Load the User Access Token, and the User Access Secret
+     */
+    public void LoadTwitterToken()
+    {
+        //Un-comment before release
+//        OAuthExample oae = new OAuthExample();
+//        OAuthTokens =  oae.GetUserAccessKeySecret();
+        //Remove before release
+        OAuthTokens = OAuthExample.DEBUGUserAccessSecret();
+    }
+
+    /**
+     * Fetches tweets matching a query
+     * @param query for which tweets need to be fetched
+     * @return an array of status objects
+     */
+    public JSONArray GetSearchResults(String query)
+    {
+        try{
+            //construct the request url
+            String URL_PARAM_SEPERATOR = "&";
+            StringBuilder url = new StringBuilder();
+            url.append("https://api.twitter.com/1.1/search/tweets.json?q=");
+            //query needs to be encoded
+            url.append(URLEncoder.encode(query, "UTF-8"));
+            url.append(URL_PARAM_SEPERATOR);
+            url.append("count=100");
+            URL navurl = new URL(url.toString());
+            HttpURLConnection huc = (HttpURLConnection) navurl.openConnection();
+            huc.setReadTimeout(5000);
+            Consumer.sign(huc);
+            huc.connect();
+            if(huc.getResponseCode()==400||huc.getResponseCode()==404||huc.getResponseCode()==429)
+            {
+                System.out.println(huc.getResponseMessage());
+                try {
+                    huc.disconnect();
+                    Thread.sleep(this.GetWaitTime("/friends/list"));
+                } catch (InterruptedException ex) {
+                    ex.printStackTrace();
+                }
+            }
+            if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
+            {
+                System.out.println(huc.getResponseMessage());
+                try {
+                    Thread.sleep(2000);
+                } catch (InterruptedException ex) {
+                    Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+                }
+            }
+            BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getInputStream()));
+            String temp;
+            StringBuilder page = new StringBuilder();
+            while( (temp = bRead.readLine())!=null)
+            {
+                page.append(temp);
+            }
+            JSONTokener jsonTokener = new JSONTokener(page.toString());
+            try {
+                JSONObject json = new JSONObject(jsonTokener);
+                JSONArray results = json.getJSONArray("statuses");
+                return results;
+            } catch (JSONException ex) {
+                Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+            }            
+        } catch (OAuthCommunicationException ex) {
+            Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthMessageSignerException ex) {
+            Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthExpectationFailedException ex) {
+            Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+        }catch(IOException ex)
+        {
+            ex.printStackTrace();
+        }
+        return null;
+    }
+
+     /**
+     * Retrieves the rate limit status of the application
+     * @return
+     */
+   public JSONObject GetRateLimitStatus()
+   {
+     try{
+            URL url = new URL("https://api.twitter.com/1.1/application/rate_limit_status.json");
+            HttpURLConnection huc = (HttpURLConnection) url.openConnection();
+            huc.setReadTimeout(5000);
+            OAuthConsumer consumer = new DefaultOAuthConsumer(utils.OAuthUtils.CONSUMER_KEY,utils.OAuthUtils.CONSUMER_SECRET);
+            consumer.setTokenWithSecret(OAuthTokens.getAccessToken(), OAuthTokens.getAccessSecret());
+            consumer.sign(huc);
+            huc.connect();
+            BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
+            StringBuffer page = new StringBuffer();
+            String temp= "";
+            while((temp = bRead.readLine())!=null)
+            {
+                page.append(temp);
+            }
+            bRead.close();
+            return (new JSONObject(page.toString()));
+        } catch (JSONException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthCommunicationException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }  catch (OAuthMessageSignerException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (OAuthExpectationFailedException ex) {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }catch(IOException ex)
+        {
+            Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
+        }
+     return null;
+   }
+
+   /**
+    * Initialize the file writer
+    * @param path of the file
+    * @param outFilename name of the file
+    */
+   public void InitializeWriters(String outFilename) {
+        try {
+            File fl = new File(outFilename);
+            if(!fl.exists())
+            {
+                fl.createNewFile();
+            }
+            /**
+             * Use UTF-8 encoding when saving files to avoid
+             * losing Unicode characters in the data
+             */
+            OutFileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilename,true),"UTF-8"));
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+   /**
+    * Close the opened filewriter to save the data
+    */
+   public void CleanupAfterFinish()
+   {
+        try {
+            OutFileWriter.close();
+        } catch (IOException ex) {
+            Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+        }
+   }
+
+   /**
+    * Writes the retrieved data to the output file
+    * @param data containing the retrived information in JSON
+    * @param user name of the user currently being written
+    */
+    public void WriteToFile(JSONArray searchResults)
+    {
+        try
+        {
+            for(int i=0;i<searchResults.length();i++)
+            {
+                try {
+                    OutFileWriter.write(searchResults.getJSONObject(i).toString());
+                    OutFileWriter.newLine();
+                } catch (JSONException ex) {
+                    Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
+                }                
+            }
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+    }
+    
+    /**
+     * Retrieves the wait time if the API Rate Limit has been hit
+     * @param api the name of the API currently being used
+     * @return the number of milliseconds to wait before initiating a new request
+     */
+    public long GetWaitTime(String api)
+    {
+        JSONObject jobj = this.GetRateLimitStatus();
+        if(jobj!=null)
+        {
+            try {
+                if(!jobj.isNull("resources"))
+                {
+                    JSONObject resourcesobj = jobj.getJSONObject("resources");
+                    JSONObject statusobj = resourcesobj.getJSONObject("statuses");
+                    JSONObject apilimit = statusobj.getJSONObject(api);
+                    int numremhits = apilimit.getInt("remaining");
+                    if(numremhits<=1)
+                    {
+                        long resettime = apilimit.getInt("reset");
+                        resettime = resettime*1000; //convert to milliseconds
+                        return resettime;
+                    }
+                }
+            } catch (JSONException ex) {
+                ex.printStackTrace();
+            }
+        }
+        return 0;
+    }
+
+    /**
+     * Creates an OR search query from the supplied terms
+     * @param queryTerms
+     * @return a String formatted as term1 OR term2
+     */
+    public String CreateORQuery(ArrayList<String> queryTerms)
+    {
+        String OR_Operator = " OR ";
+        StringBuffer querystr = new StringBuffer();
+        int count = 1;
+        for(String term:queryTerms)
+        {
+            if(count==1)
+            {
+                querystr.append(term);
+            }
+            else
+            {
+                querystr.append(OR_Operator).append(term);
+            }
+        }
+        return querystr.toString();
+    }
+
+    public static void main(String[] args)
+    {
+        RESTSearchExample rse = new RESTSearchExample();
+        ArrayList<String> queryterms = new ArrayList<String>();        
+        String outfilename = rse.DEF_FILENAME;
+        if(args!=null)
+        {
+            if(args.length>0)
+            {
+                for(int i=0;i<args.length;i++)
+                {
+                    queryterms.add(args[i]);
+                }
+            }
+            else
+            {
+                queryterms.add(rse.query);
+            }
+        }
+        rse.LoadTwitterToken();
+        rse.Consumer = rse.GetConsumer();
+        System.out.println(rse.GetRateLimitStatus());
+        rse.InitializeWriters(outfilename);
+        JSONArray results = rse.GetSearchResults(rse.CreateORQuery(queryterms));
+        if(results!=null)
+        {
+            rse.WriteToFile(results);
+        }
+        rse.CleanupAfterFinish();
+    }
+}
diff --git a/src/Chapter2/streamingapi/StreamingApiExample.java b/src/Chapter2/streamingapi/StreamingApiExample.java
new file mode 100644
index 0000000..8abfff4
--- /dev/null
+++ b/src/Chapter2/streamingapi/StreamingApiExample.java
@@ -0,0 +1,372 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.streamingapi;
+
+import Chapter2.support.OAuthTokenSecret;
+import Chapter2.openauthentication.OAuthExample;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
+import oauth.signpost.exception.OAuthCommunicationException;
+import oauth.signpost.exception.OAuthExpectationFailedException;
+import oauth.signpost.exception.OAuthMessageSignerException;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.HttpEntity;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.params.CoreConnectionPNames;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import utils.OAuthUtils;
+
+public class StreamingApiExample
+{
+    OAuthTokenSecret OAuthToken;
+    final int RECORDS_TO_PROCESS = 1000;
+    final int MAX_GEOBOXES = 25;
+    final int MAX_KEYWORDS = 400;
+    final int MAX_USERS = 5000;
+    HashSet<String> Keywords;
+    HashSet<String> Geoboxes;
+    HashSet<String> Userids;
+    final String CONFIG_FILE_PATH = "streaming/streaming.config";
+    final String DEF_OUTPATH = "streaming/";
+
+    /**
+     * Loads the Twitter access token and secret for a user
+     */    
+    public void LoadTwitterToken()
+    {
+//        OAuthExample oae = new OAuthExample();
+//        OAuthToken =  oae.GetUserAccessKeySecret();
+        OAuthToken = OAuthExample.DEBUGUserAccessSecret();
+    }
+
+    /**
+     * Creates a connection to the Streaming Filter API
+     * @param baseUrl the URL for Twitter Filter API
+     * @param outFilePath Location to place the exported file
+     */
+    public void CreateStreamingConnection(String baseUrl, String outFilePath)
+    {
+        HttpClient httpClient = new DefaultHttpClient();
+        httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, new Integer(90000));
+        //Step 1: Initialize OAuth Consumer
+        OAuthConsumer consumer = new CommonsHttpOAuthConsumer(OAuthUtils.CONSUMER_KEY,OAuthUtils.CONSUMER_SECRET);
+        consumer.setTokenWithSecret(OAuthToken.getAccessToken(),OAuthToken.getAccessSecret());
+        //Step 2: Create a new HTTP POST request and set parameters
+        HttpPost httppost = new HttpPost(baseUrl);
+        try {            
+            httppost.setEntity(new UrlEncodedFormEntity(CreateRequestBody(), "UTF-8"));
+        } catch (UnsupportedEncodingException ex) {
+            ex.printStackTrace();
+        }
+        try {
+             //Step 3: Sign the request
+                consumer.sign(httppost);
+            } catch (OAuthMessageSignerException ex) {
+                ex.printStackTrace();
+            } catch (OAuthExpectationFailedException ex) {
+                ex.printStackTrace();
+            } catch (OAuthCommunicationException ex) {
+                ex.printStackTrace();
+            }
+        HttpResponse response;
+        InputStream is = null;
+        try {
+             //Step 4: Connect to the API
+                response = httpClient.execute(httppost);
+                if (response.getStatusLine().getStatusCode()!= HttpStatus.SC_OK)
+                {
+                    throw new IOException("Got status " +response.getStatusLine().getStatusCode());
+                }
+                else
+                {
+                    System.out.println(OAuthToken.getAccessToken()+ ": Processing from " + baseUrl);
+                    HttpEntity entity = response.getEntity();
+                    try {
+                        is = entity.getContent();
+                    } catch (IOException ex) {
+                        ex.printStackTrace();
+                    } catch (IllegalStateException ex) {
+                        ex.printStackTrace();
+                    }
+                    //Step 5: Process the incoming Tweet Stream
+                    this.ProcessTwitterStream(is, outFilePath);
+                }
+         } catch (IOException ex) {
+            ex.printStackTrace();
+        }finally {
+            // Abort the method, otherwise releaseConnection() will
+            // attempt to finish reading the never-ending response.
+            // These methods do not throw exceptions.
+            if(is!=null)
+            {
+                try {
+                    is.close();
+                } catch (IOException ex) {
+                    ex.printStackTrace();
+                }
+            }
+        }
+    }
+
+    /**
+     *  Processes a stream of tweets and writes them to a file one tweet per line. Each tweet here is represented by a JSON document.
+     * @param is input stream already connected to the streaming API
+     * @param outFilePath file to put the collected tweets in
+     * @throws InterruptedException
+     * @throws IOException
+     */
+    public void ProcessTwitterStream(InputStream is, String outFilePath)
+    {
+        BufferedWriter bwrite = null;
+        try {
+            JSONTokener jsonTokener = new JSONTokener(new InputStreamReader(is, "UTF-8"));
+            ArrayList<JSONObject> rawtweets = new ArrayList<JSONObject>();
+            int nooftweetsuploaded = 0;
+            while (true) {
+                try {                    
+                    JSONObject temp = new JSONObject(jsonTokener);                    
+                    rawtweets.add(temp);
+//                    System.out.println(temp);
+                    if (rawtweets.size() >= RECORDS_TO_PROCESS)
+                    {
+                        Calendar cal = Calendar.getInstance();
+                        String filename = outFilePath + "tweets_" + cal.getTimeInMillis() + ".json";
+                        bwrite = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), "UTF-8"));
+                        nooftweetsuploaded += RECORDS_TO_PROCESS;
+                        //Write the collected tweets to a file
+                        for (JSONObject jobj : rawtweets) {
+                            bwrite.write(jobj.toString());
+                            bwrite.newLine();
+                        }
+                        System.out.println("Written "+nooftweetsuploaded+" records so far");
+                        bwrite.close();
+                        rawtweets.clear();
+                    }
+                } catch (JSONException ex) {
+                    ex.printStackTrace();
+                }                
+            }
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        } 
+    }
+
+    public static void main(String[] args)
+    {
+        StreamingApiExample sae = new StreamingApiExample();
+        sae.LoadTwitterToken();
+        //load parameters from a TSV file
+        String filename = sae.CONFIG_FILE_PATH;
+        String outfilepath = sae.DEF_OUTPATH;
+        if(args!=null)
+        {
+            if(args.length>0)
+            {
+                filename = args[0];
+            }
+            if(args.length>1)
+            {
+                File fl = new File(args[1]);
+                if(fl.exists()&&fl.isDirectory())
+                {
+                    outfilepath = args[1];
+                }
+            }
+        }
+        sae.ReadParameters(filename);
+        sae.CreateStreamingConnection("https://stream.twitter.com/1.1/statuses/filter.json", outfilepath);
+    }
+
+    /**
+     * Reads the file and loads the parameters to be crawled. Expects that the parameters are tab separated values and the
+     * @param filename
+     */
+    public void ReadParameters(String filename)
+    {
+        BufferedReader br = null;
+        try {
+            br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8"));
+            String temp = "";
+            int count = 1;
+            if(Userids==null)
+            {
+                Userids = new HashSet<String>();
+            }
+            if(Geoboxes==null)
+            {
+                Geoboxes = new HashSet<String>();
+            }
+            if(Keywords==null)
+            {
+                Keywords = new HashSet<String>();
+            }
+            while((temp = br.readLine())!=null)
+            {
+                if(!temp.isEmpty())
+                {
+                    if(count==1)
+                    {                        
+                        String[] keywords = temp.split("\t");
+                        HashSet<String> temptags = new HashSet<String>();
+                        for(String word:keywords)
+                        {
+                            if(!temptags.contains(word))
+                            {
+                                temptags.add(word);
+                            }
+                        }
+                        FilterKeywords(temptags);
+                    }
+                    else
+                    if(count==2)
+                    {
+                        String[] geoboxes = temp.split("\t");
+                        HashSet<String> tempboxes = new HashSet<String>();
+                        for(String box:geoboxes)
+                        {
+                            if(!tempboxes.contains(box))
+                            {
+                                tempboxes.add(box);
+                            }
+                        }
+                        FilterGeoboxes(tempboxes);
+                    }
+                    else
+                    if(count==3)
+                    {
+                        String[] userids = temp.split("\t");
+                        HashSet<String> tempids = new HashSet<String>();
+                        for(String id:userids)
+                        {
+                            if(!tempids.contains(id))
+                            {
+                                tempids.add(id);
+                            }
+                        }
+                        FilterUserids(tempids);
+                    }
+                    count++;
+                }
+            }
+        } catch (IOException ex) {
+            ex.printStackTrace();
+        }
+        finally{
+            try {
+                br.close();
+            } catch (IOException ex) {
+                ex.printStackTrace();
+            }
+        }
+    }
+
+    private void FilterUserids(HashSet<String> userids)
+    {
+        if(userids!=null)
+        {
+            int maxsize = MAX_USERS;
+            if(userids.size()<maxsize)
+            {
+                maxsize = userids.size();
+            }
+            for(String id:userids)
+            {
+                Userids.add(id);
+            }
+        }
+    }
+
+    private void FilterGeoboxes(HashSet<String> geoboxes)
+    {
+        if(geoboxes!=null)
+        {
+            int maxsize = MAX_GEOBOXES;
+            if(geoboxes.size()<maxsize)
+            {
+                maxsize = geoboxes.size();
+            }
+            for(String box:geoboxes)
+            {
+                Geoboxes.add(box);
+            }
+        }
+    }
+    /**
+     * Keep only the maximum permitted number of parameters for a connection. Ignoring the rest.
+     * This can be extended to create multiple sets to be crawled by different threads.
+     */
+    private void FilterKeywords(HashSet<String> hashtags)
+    {          
+        if(hashtags!=null)
+        {
+            int maxsize = MAX_KEYWORDS;
+            if(hashtags.size()<maxsize)
+            {
+                maxsize = hashtags.size();
+            }
+            for(String tag:hashtags)
+            {
+                Keywords.add(tag);
+            }
+        }
+             
+    }
+
+     private List<NameValuePair> CreateRequestBody()
+     {
+        List<NameValuePair> params = new ArrayList<NameValuePair>();
+        if(Userids != null&&Userids.size()>0)
+        {
+            params.add(CreateNameValuePair("follow", Userids));
+            System.out.println("userids = "+Userids);
+        }
+        if (Geoboxes != null&&Geoboxes.size()>0) {
+            params.add(CreateNameValuePair("locations", Geoboxes));
+            System.out.println("locations = "+Geoboxes);
+
+        }
+        if (Keywords != null&&Keywords.size()>0) {
+            params.add(CreateNameValuePair("track", Keywords));
+            System.out.println("keywords = "+Keywords);
+        }
+        return params;
+    }
+
+    private NameValuePair CreateNameValuePair(String name, Collection<String> items)
+    {
+        StringBuilder sb = new StringBuilder();
+        boolean needComma = false;
+        for (String item : items) {
+            if (needComma) {
+                sb.append(',');
+            }
+            needComma = true;
+            sb.append(item);
+        }
+        return new BasicNameValuePair(name, sb.toString());
+    }
+}
diff --git a/src/Chapter2/support/APIType.java b/src/Chapter2/support/APIType.java
new file mode 100644
index 0000000..94449f8
--- /dev/null
+++ b/src/Chapter2/support/APIType.java
@@ -0,0 +1,12 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.support;
+
+public class APIType
+{
+    public static String USER_TIMELINE = "/statuses/user_timeline";
+    public static String FOLLOWERS = "/followers/list";
+    public static String FRIENDS = "/friends/list";
+    public static String USER_PROFILE = "/users/show";
+}
diff --git a/src/Chapter2/support/InfoType.java b/src/Chapter2/support/InfoType.java
new file mode 100644
index 0000000..42b0334
--- /dev/null
+++ b/src/Chapter2/support/InfoType.java
@@ -0,0 +1,12 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.support;
+
+public class InfoType
+{
+    public static final int PROFILE_INFO = 0;
+    public static final int FOLLOWER_INFO = 1;
+    public static final int FRIEND_INFO = 2;
+    public static final int STATUSES_INFO = 3;
+}
diff --git a/src/Chapter2/support/Location.java b/src/Chapter2/support/Location.java
new file mode 100644
index 0000000..7f6234f
--- /dev/null
+++ b/src/Chapter2/support/Location.java
@@ -0,0 +1,28 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package Chapter2.support;
+
+/**
+ *
+ * @author shamanth
+ */
+public class Location
+{
+    public Double latitude;
+    public Double longitude;
+
+    public Location(Double lat,Double lng)
+    {
+        latitude = lat;
+        longitude  = lng;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Latitude: "+latitude+" & Longitude: "+longitude;
+    }
+}
diff --git a/src/Chapter2/support/OAuthTokenSecret.java b/src/Chapter2/support/OAuthTokenSecret.java
new file mode 100644
index 0000000..8fee4a8
--- /dev/null
+++ b/src/Chapter2/support/OAuthTokenSecret.java
@@ -0,0 +1,38 @@
+/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
+ * @author shamanth
+ */
+package Chapter2.support;
+
+public class OAuthTokenSecret
+{
+    String UserAccessToken;
+    String UserAccessSecret;
+
+    public String getAccessSecret() {
+        return UserAccessSecret;
+    }
+
+    public void setAccessSecret(String AccessSecret) {
+        this.UserAccessSecret = AccessSecret;
+    }
+
+    public String getAccessToken() {
+        return UserAccessToken;
+    }
+
+    public void setAccessToken(String AccessToken) {
+        this.UserAccessToken = AccessToken;
+    }
+
+    public OAuthTokenSecret(String token,String secret)
+    {
+        this.setAccessToken(token);
+        this.setAccessSecret(secret);
+    }
+
+    @Override
+    public String toString()
+    {
+       return "Access Token: "+getAccessToken()+" Access Secret: "+getAccessSecret();
+    }
+}