summaryrefslogtreecommitdiff
path: root/src/Chapter2
diff options
context:
space:
mode:
Diffstat (limited to 'src/Chapter2')
-rw-r--r--src/Chapter2/Location/LocationTranslationExample.java124
-rw-r--r--src/Chapter2/openauthentication/OAuthExample.java79
-rw-r--r--src/Chapter2/restapi/RESTApiExample.java676
-rw-r--r--src/Chapter2/restapi/RESTSearchExample.java311
-rw-r--r--src/Chapter2/streamingapi/StreamingApiExample.java372
-rw-r--r--src/Chapter2/support/APIType.java12
-rw-r--r--src/Chapter2/support/InfoType.java12
-rw-r--r--src/Chapter2/support/Location.java28
-rw-r--r--src/Chapter2/support/OAuthTokenSecret.java38
9 files changed, 0 insertions, 1652 deletions
diff --git a/src/Chapter2/Location/LocationTranslationExample.java b/src/Chapter2/Location/LocationTranslationExample.java
deleted file mode 100644
index 69178dc..0000000
--- a/src/Chapter2/Location/LocationTranslationExample.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.Location;
-
-import Chapter2.support.Location;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.net.URLEncoder;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.json.JSONArray;
-import org.json.JSONException;
-
-public class LocationTranslationExample
-{
-
- /**
- * Translates a location string to coordinates using the database or Nominatim Service
- * @param loc
- * @return
- */
- public Location TranslateLoc(String loc)
- {
- if(loc!=null&&!loc.isEmpty())
- {
- String encodedLoc="";
- try {
- //Step 1: Encode the location name
- encodedLoc = URLEncoder.encode(loc, "UTF-8");
- } catch (UnsupportedEncodingException ex) {
- Logger.getLogger(LocationTranslationExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- //Step 2: Create a get request to MapQuest API with the name of the location
- String url= "http://open.mapquestapi.com/nominatim/v1/search?q="+encodedLoc+"&format=json";
- String page = ReadHTML(url);
- if(page!=null)
- {
- try{
- JSONArray results = new JSONArray(page);
- if(results.length()>0)
- {
- //Step 3: Read and extract the coordinates of the location as a JSONObject
- Location loca = new Location(results.getJSONObject(0).getDouble("lat"),results.getJSONObject(0).getDouble("lon"));
- return loca;
- }
- }catch(JSONException ex)
- {
- Logger.getLogger(LocationTranslationExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- }
- return null;
- }
-
- /**
- * Extracts the html content of a URL
- * @param url
- * @return html page
- */
- public String ReadHTML(String url)
- {
- URLConnection conn = null;
- URL theURL = null;
- try
- {
- theURL = new URL(url);
- }
- catch ( MalformedURLException e)
- {
- System.out.println("Bad URL: " + theURL);
- return null;
- }
- String page = "";
- try
- {
- conn = theURL.openConnection();
- HttpURLConnection huc = (HttpURLConnection) conn;
- conn.setConnectTimeout(2000);
- huc.setRequestProperty("User-Agent", "Mozilla/4.5");
- //Set your email address in the request so MapQuest knows how to reach you in the event of problems
- huc.setRequestProperty("Email", "twitterdataanalytics@gmail.com");
- if(huc.getResponseCode()>=400&&huc.getResponseCode()<=404)
- {
- return null;
- }
- conn.connect();
- BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) conn.getContent()));
- String temp=null;
- while( (temp= bRead.readLine())!=null)
- {
- page = page+"\n"+temp;
- }
- bRead.close();
- }
- catch (IOException e) {
- //System.out.print("ReadHTML IO Error:" + e.getMessage()+" \n");
- return null;
- }
- return page;
- }
-
- public static void main(String[] args)
- {
- LocationTranslationExample lte = new LocationTranslationExample();
- if(args!=null)
- {
- if(args.length>0)
- {
- for(int i=0;i<args.length;i++)
- {
- System.out.println(lte.TranslateLoc(args[i]).toString());
- }
- }
- }
- }
-}
diff --git a/src/Chapter2/openauthentication/OAuthExample.java b/src/Chapter2/openauthentication/OAuthExample.java
deleted file mode 100644
index 34ca3a1..0000000
--- a/src/Chapter2/openauthentication/OAuthExample.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.openauthentication;
-
-import Chapter2.support.OAuthTokenSecret;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import oauth.signpost.OAuth;
-import oauth.signpost.OAuthConsumer;
-import oauth.signpost.OAuthProvider;
-import oauth.signpost.basic.DefaultOAuthProvider;
-import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
-import oauth.signpost.exception.OAuthCommunicationException;
-import oauth.signpost.exception.OAuthExpectationFailedException;
-import oauth.signpost.exception.OAuthMessageSignerException;
-import oauth.signpost.exception.OAuthNotAuthorizedException;
-import utils.Configuration;
-
-public class OAuthExample
-{
- public OAuthTokenSecret GetUserAccessKeySecret()
- {
- try {
- //consumer key for Twitter Data Analytics application
- if(Configuration.CONSUMER_KEY.isEmpty())
- {
- System.out.println("Register an application and copy the consumer key into the configuration file.");
- return null;
- }
- if(Configuration.CONSUMER_SECRET.isEmpty())
- {
- System.out.println("Register an application and copy the consumer secret into the configuration file.");
- return null;
- }
- OAuthConsumer consumer = new CommonsHttpOAuthConsumer(Configuration.CONSUMER_KEY,Configuration.CONSUMER_SECRET);
- OAuthProvider provider = new DefaultOAuthProvider(Configuration.REQUEST_TOKEN_URL, Configuration.ACCESS_TOKEN_URL, Configuration.AUTHORIZE_URL);
- String authUrl = provider.retrieveRequestToken(consumer, OAuth.OUT_OF_BAND);
- System.out.println("Now visit:\n" + authUrl + "\n and grant this app authorization");
- System.out.println("Enter the PIN code and hit ENTER when you're done:");
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
- String pin = br.readLine();
- System.out.println("Fetching access token from Twitter");
- provider.retrieveAccessToken(consumer,pin);
- String accesstoken = consumer.getToken();
- String accesssecret = consumer.getTokenSecret();
- OAuthTokenSecret tokensecret = new OAuthTokenSecret(accesstoken,accesssecret);
- return tokensecret;
- } catch (OAuthNotAuthorizedException ex) {
- ex.printStackTrace();
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- } catch(IOException ex)
- {
- ex.printStackTrace();
- }
- return null;
- }
-
- public static OAuthTokenSecret DEBUGUserAccessSecret()
- {
- String accesstoken = "1262619914-tcCPB1SyXy3BMuui9OAhprcPmqg3z2csSjDSCNY";
- String accesssecret = "cXXO0qFLBjLXGtE97pnf5Vv1RZGxZ2FZ97wCYiaVU";
- OAuthTokenSecret tokensecret = new OAuthTokenSecret(accesstoken,accesssecret);
- return tokensecret;
- }
-
- public static void main(String[] args)
- {
- OAuthExample aue = new OAuthExample();
- OAuthTokenSecret tokensecret = aue.GetUserAccessKeySecret();
- System.out.println(tokensecret.toString());
- }
-}
diff --git a/src/Chapter2/restapi/RESTApiExample.java b/src/Chapter2/restapi/RESTApiExample.java
deleted file mode 100644
index 3d6074b..0000000
--- a/src/Chapter2/restapi/RESTApiExample.java
+++ /dev/null
@@ -1,676 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.restapi;
-
-import Chapter2.support.APIType;
-import Chapter2.support.OAuthTokenSecret;
-import Chapter2.openauthentication.OAuthExample;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import oauth.signpost.OAuthConsumer;
-import oauth.signpost.basic.DefaultOAuthConsumer;
-import oauth.signpost.exception.OAuthCommunicationException;
-import oauth.signpost.exception.OAuthExpectationFailedException;
-import oauth.signpost.exception.OAuthMessageSignerException;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-public class RESTApiExample
-{
- //file handlers to store the collected user information
- BufferedWriter OutFileWriter;
- OAuthTokenSecret OAuthTokens;
- /**
- * name of the file containing a list of users
- */
- final String DEF_FILENAME = "users.txt";
- final String DEF_OUTFILENAME = "restapiresults.json";
- ArrayList<String> Usernames = new ArrayList<String>();
- OAuthConsumer Consumer;
-
- /**
- * Creates a OAuthConsumer with the current consumer & user access tokens and secrets
- * @return consumer
- */
- public OAuthConsumer GetConsumer()
- {
- OAuthConsumer consumer = new DefaultOAuthConsumer(utils.Configuration.CONSUMER_KEY,utils.Configuration.CONSUMER_SECRET);
- consumer.setTokenWithSecret(OAuthTokens.getAccessToken(),OAuthTokens.getAccessSecret());
- return consumer;
- }
-
- /**
- * Reads the file and loads the users in the file to be crawled
- * @param filename
- */
- public void ReadUsers(String filename)
- {
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8"));
- String temp = "";
- while((temp = br.readLine())!=null)
- {
- if(!temp.isEmpty())
- {
- Usernames.add(temp);
- }
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- finally{
- try {
- br.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
- }
-
- /**
- * Load the User Access Token, and the User Access Secret
- */
- public void LoadTwitterToken()
- {
- //Un-comment before release
-// OAuthExample oae = new OAuthExample();
-// OAuthTokens = oae.GetUserAccessKeySecret();
- //Remove before release
- OAuthTokens = OAuthExample.DEBUGUserAccessSecret();
- }
-
- public static void main(String[] args)
- {
- RESTApiExample rae = new RESTApiExample();
- rae.LoadTwitterToken();
- rae.Consumer = rae.GetConsumer();
-// System.out.println(rae.GetStatuses("twtanalyticsbk"));
- System.out.println(rae.GetRateLimitStatus());
-// int apicode = InfoType.PROFILE_INFO;
-// String infilename = rae.DEF_FILENAME;
-// String outfilename = rae.DEF_OUTFILENAME;
-// if(args!=null)
-// {
-// if(args.length>2)
-// {
-// apicode = Integer.parseInt(args[2]);
-// outfilename = args[1];
-// infilename = args[0];
-// }
-// if(args.length>1)
-// {
-// outfilename = args[1];
-// infilename = args[0];
-// }
-// else
-// if(args.length>0)
-// {
-// infilename = args[0];
-// }
-// }
-// rae.InitializeWriters(outfilename);
-// rae.ReadUsers(infilename);
-// if(apicode!=InfoType.PROFILE_INFO&&apicode!=InfoType.FOLLOWER_INFO&&apicode!=InfoType.FRIEND_INFO&&apicode!=InfoType.STATUSES_INFO)
-// {
-// System.out.println("Invalid API type: Use 0 for Profile, 1 for Followers, 2 for Friends, and 3 for Statuses");
-// System.exit(0);
-// }
-// if(rae.Usernames.size()>0)
-// {
-// //TO-DO: Print the possible API types and get user selection to crawl the users.
-// rae.LoadTwitterToken();
-// for(String user:rae.Usernames)
-// {
-// if(apicode==InfoType.PROFILE_INFO)
-// {
-// JSONObject jobj = rae.GetProfile(user);
-// if(jobj!=null&&jobj.length()==0)
-// {
-// rae.WriteToFile(user, jobj.toString());
-// }
-// }
-// else
-// if(apicode==InfoType.FRIEND_INFO)
-// {
-// JSONArray statusarr = rae.GetFriends(user);
-// if(statusarr.length()>0)
-// {
-// rae.WriteToFile(user, statusarr.toString());
-// }
-// }
-// else
-// if(apicode == InfoType.FOLLOWER_INFO)
-// {
-// JSONArray statusarr = rae.GetFollowers(user);
-// if(statusarr.length()>0)
-// {
-// rae.WriteToFile(user, statusarr.toString());
-// }
-// }
-// else
-// if(apicode == InfoType.STATUSES_INFO)
-// {
-// JSONArray statusarr = rae.GetStatuses(user);
-// if(statusarr.length()>0)
-// {
-// rae.GetStatuses(user);
-// }
-// }
-// }
-// }
-//// now you can close the files as all the threads have finished
-// rae.CleanupAfterFinish();
- }
-
- /**
- * Retrieves the rate limit status of the application
- * @return
- */
- public JSONObject GetRateLimitStatus()
- {
- try{
- URL url = new URL("https://api.twitter.com/1.1/application/rate_limit_status.json");
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- Consumer.sign(huc);
- huc.connect();
- BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
- StringBuffer page = new StringBuffer();
- String temp= "";
- while((temp = bRead.readLine())!=null)
- {
- page.append(temp);
- }
- bRead.close();
- return (new JSONObject(page.toString()));
- } catch (JSONException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthCommunicationException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthMessageSignerException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthExpectationFailedException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }catch(IOException ex)
- {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- return null;
- }
-
- /**
- * Initialize the file writer
- * @param path of the file
- * @param outFilename name of the file
- */
- public void InitializeWriters(String outFilename) {
- try {
- File fl = new File(outFilename);
- if(!fl.exists())
- {
- fl.createNewFile();
- }
- /**
- * Use UTF-8 encoding when saving files to avoid
- * losing Unicode characters in the data
- */
- OutFileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilename,true),"UTF-8"));
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * Close the opened filewriter to save the data
- */
- public void CleanupAfterFinish()
- {
- try {
- OutFileWriter.close();
- } catch (IOException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- /**
- * Writes the retrieved data to the output file
- * @param data containing the retrived information in JSON
- * @param user name of the user currently being written
- */
- public void WriteToFile(String user, String data)
- {
- try
- {
- OutFileWriter.write(data);
- OutFileWriter.newLine();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * Retrives the profile information of the user
- * @param username of the user whose profile needs to be retrieved
- * @return the profile information as a JSONObject
- */
- public JSONObject GetProfile(String username)
- {
- BufferedReader bRead = null;
- JSONObject profile = null;
- try {
- System.out.println("Processing profile of "+username);
- boolean flag = true;
- URL url = new URL("https://api.twitter.com/1.1/users/show.json?screen_name="+username);
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- // Step 2: Sign the request using the OAuth Secret
- Consumer.sign(huc);
- huc.connect();
- if(huc.getResponseCode()==404||huc.getResponseCode()==401)
- {
- System.out.println(huc.getResponseMessage());
- }
- else
- if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
- {
- try {
- huc.disconnect();
- System.out.println(huc.getResponseMessage());
- Thread.sleep(3000);
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- else
- // Step 3: If the requests have been exhausted, then wait until the quota is renewed
- if(huc.getResponseCode()==429)
- {
- try {
- huc.disconnect();
- Thread.sleep(this.GetWaitTime("/users/show/:id"));
- flag = false;
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- if(!flag)
- {
- //recreate the connection because something went wrong the first time.
- huc.connect();
- }
- StringBuilder content=new StringBuilder();
- if(flag)
- {
- bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
- String temp= "";
- while((temp = bRead.readLine())!=null)
- {
- content.append(temp);
- }
- }
- huc.disconnect();
- try {
- profile = new JSONObject(content.toString());
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- return profile;
- }
-
- /**
- * Retrieves the followers of a user
- * @param username the name of the user whose followers need to be retrieved
- * @return a list of user objects corresponding to the followers of the user
- */
- public JSONArray GetFollowers(String username)
- {
- BufferedReader bRead = null;
- JSONArray followers = new JSONArray();
- try {
- System.out.println(" followers user = "+username);
- long cursor = -1;
- while(true)
- {
- if(cursor==0)
- {
- break;
- }
- // Step 1: Create the APi request using the supplied username
- URL url = new URL("https://api.twitter.com/1.1/followers/list.json?screen_name="+username+"&cursor=" + cursor);
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- // Step 2: Sign the request using the OAuth Secret
- Consumer.sign(huc);
- huc.connect();
- if(huc.getResponseCode()==400||huc.getResponseCode()==404)
- {
- System.out.println(huc.getResponseMessage());
- break;
- }
- else
- if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503||huc.getResponseCode()==504)
- {
- try{
- System.out.println(huc.getResponseMessage());
- huc.disconnect();
- Thread.sleep(3000);
- continue;
- } catch (InterruptedException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- else
- // Step 3: If the requests have been exhausted, then wait until the quota is renewed
- if(huc.getResponseCode()==429)
- {
- try {
- huc.disconnect();
- Thread.sleep(this.GetWaitTime("/followers/list"));
- continue;
- } catch (InterruptedException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- // Step 4: Retrieve the followers list from Twitter
- bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
- StringBuilder content = new StringBuilder();
- String temp = "";
- while((temp = bRead.readLine())!=null)
- {
- content.append(temp);
- }
- try {
- JSONObject jobj = new JSONObject(content.toString());
- // Step 5: Retrieve the token for the next request
- cursor = jobj.getLong("next_cursor");
- JSONArray idlist = jobj.getJSONArray("users");
- if(idlist.length()==0)
- {
- break;
- }
- for(int i=0;i<idlist.length();i++)
- {
- followers.put(idlist.getJSONObject(i));
- }
- } catch (JSONException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- return followers;
- }
-
- /**
- * Retrieved the status messages of a user
- * @param username the name of the user whose status messages need to be retrieved
- * @return a list of status messages
- */
- public JSONArray GetStatuses(String username)
- {
- BufferedReader bRead = null;
- //Get the maximum number of tweets possible in a single page 200
- int tweetcount = 200;
- //Include include_rts because it is counted towards the limit anyway.
- boolean include_rts = true;
- JSONArray statuses = new JSONArray();
- try {
- System.out.println("Processing status messages of "+username);
- long maxid = 0;
- while(true)
- {
- URL url = null;
- if(maxid==0)
- {
- url = new URL("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=" + username+"&include_rts="+include_rts+"&count="+tweetcount);
- }
- else
- {
- //use max_id to get the tweets in the next page. Use max_id-1 to avoid getting redundant tweets.
- url = new URL("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=" + username+"&include_rts="+include_rts+"&count="+tweetcount+"&max_id="+(maxid-1));
- }
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- Consumer.sign(huc);
- huc.connect();
- if(huc.getResponseCode()==400||huc.getResponseCode()==404)
- {
- System.out.println(huc.getResponseCode());
- break;
- }
- else
- if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
- {
- try {System.out.println(huc.getResponseCode());
- Thread.sleep(3000);
- } catch (InterruptedException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- else
- // Step 3: If the requests have been exhausted, then wait until the quota is renewed
- if(huc.getResponseCode()==429)
- {
- try {
- huc.disconnect();
- Thread.sleep(this.GetWaitTime("/statuses/user_timeline"));
- continue;
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getInputStream()));
- StringBuilder content = new StringBuilder();
- String temp = "";
- while((temp = bRead.readLine())!=null)
- {
- content.append(temp);
- }
- try {
- JSONArray statusarr = new JSONArray(content.toString());
- if(statusarr.length()==0)
- {
- break;
- }
- for(int i=0;i<statusarr.length();i++)
- {
- JSONObject jobj = statusarr.getJSONObject(i);
- statuses.put(jobj);
- //Get the max_id to get the next batch of tweets
- if(!jobj.isNull("id"))
- {
- maxid = jobj.getLong("id");
- }
- }
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- }
- System.out.println(statuses.length());
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- return statuses;
- }
-
- /**
- * Retrieves the friends of a user
- * @param username the name of the user whose friends need to be fetched
- * @return a list of user objects who are friends of the user
- */
- public JSONArray GetFriends(String username)
- {
- BufferedReader bRead = null;
- JSONArray friends = new JSONArray();
- try {
- System.out.println("Processing friends of "+username);
- long cursor = -1;
- while(true)
- {
- if(cursor==0)
- {
- break;
- }
- // Step 1: Create the APi request using the supplied username
- URL url = new URL("https://api.twitter.com/1.1/friends/list.json?screen_name="+username+"&cursor="+cursor);
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- //Step 2: Sign the request using the OAuth Secret
- Consumer.sign(huc);
- huc.connect();
- if(huc.getResponseCode()==400||huc.getResponseCode()==401)
- {
- System.out.println(huc.getResponseMessage());
- break;
- }
- else
- if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
- {
- try {
- System.out.println(huc.getResponseMessage());
- Thread.sleep(3000);
- continue;
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- else
- // Step 3: If the requests have been exhausted, then wait until the quota is renewed
- if(huc.getResponseCode()==429)
- {
- try {
- huc.disconnect();
- Thread.sleep(this.GetWaitTime("/friends/list"));
- continue;
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- // Step 4: Retrieve the friends list from Twitter
- bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
- StringBuilder content = new StringBuilder();
- String temp = "";
- while((temp = bRead.readLine())!=null)
- {
- content.append(temp);
- }
- try {
- JSONObject jobj = new JSONObject(content.toString());
- // Step 5: Retrieve the token for the next request
- cursor = jobj.getLong("next_cursor");
- JSONArray userlist = jobj.getJSONArray("users");
- if(userlist.length()==0)
- {
- break;
- }
- for(int i=0;i<userlist.length();i++)
- {
- friends.put(userlist.get(i));
- }
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- huc.disconnect();
- }
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- return friends;
- }
-
- /**
- * Retrieves the wait time if the API Rate Limit has been hit
- * @param api the name of the API currently being used
- * @return the number of milliseconds to wait before initiating a new request
- */
- public long GetWaitTime(String api)
- {
- JSONObject jobj = this.GetRateLimitStatus();
- if(jobj!=null)
- {
- try {
- if(!jobj.isNull("resources"))
- {
- JSONObject resourcesobj = jobj.getJSONObject("resources");
- JSONObject apilimit = null;
- if(api.equals(APIType.USER_TIMELINE))
- {
- JSONObject statusobj = resourcesobj.getJSONObject("statuses");
- apilimit = statusobj.getJSONObject(api);
- }
- else
- if(api.equals(APIType.FOLLOWERS))
- {
- JSONObject followersobj = resourcesobj.getJSONObject("followers");
- apilimit = followersobj.getJSONObject(api);
- }
- else
- if(api.equals(APIType.FRIENDS))
- {
- JSONObject friendsobj = resourcesobj.getJSONObject("friends");
- apilimit = friendsobj.getJSONObject(api);
- }
- else
- if(api.equals(APIType.USER_PROFILE))
- {
- JSONObject userobj = resourcesobj.getJSONObject("users");
- apilimit = userobj.getJSONObject(api);
- }
- int numremhits = apilimit.getInt("remaining");
- if(numremhits<=1)
- {
- long resettime = apilimit.getInt("reset");
- resettime = resettime*1000; //convert to milliseconds
- return resettime;
- }
- }
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- }
- return 0;
- }
-}
diff --git a/src/Chapter2/restapi/RESTSearchExample.java b/src/Chapter2/restapi/RESTSearchExample.java
deleted file mode 100644
index e9a5dd7..0000000
--- a/src/Chapter2/restapi/RESTSearchExample.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.restapi;
-
-import Chapter2.support.OAuthTokenSecret;
-import Chapter2.openauthentication.OAuthExample;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import oauth.signpost.OAuthConsumer;
-import oauth.signpost.basic.DefaultOAuthConsumer;
-import oauth.signpost.exception.OAuthCommunicationException;
-import oauth.signpost.exception.OAuthExpectationFailedException;
-import oauth.signpost.exception.OAuthMessageSignerException;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-
-public class RESTSearchExample
-{
- BufferedWriter OutFileWriter;
- OAuthTokenSecret OAuthTokens;
- OAuthConsumer Consumer;
- String query = "#protest";
- String DEF_FILENAME = "searchresults.json";
-
- /**
- * Creates a OAuthConsumer with the current consumer & user access tokens and secrets
- * @return consumer
- */
- public OAuthConsumer GetConsumer()
- {
- OAuthConsumer consumer = new DefaultOAuthConsumer(utils.Configuration.CONSUMER_KEY,utils.Configuration.CONSUMER_SECRET);
- consumer.setTokenWithSecret(OAuthTokens.getAccessToken(), OAuthTokens.getAccessSecret());
- return consumer;
- }
-
- /**
- * Load the User Access Token, and the User Access Secret
- */
- public void LoadTwitterToken()
- {
- //Un-comment before release
-// OAuthExample oae = new OAuthExample();
-// OAuthTokens = oae.GetUserAccessKeySecret();
- //Remove before release
- OAuthTokens = OAuthExample.DEBUGUserAccessSecret();
- }
-
- /**
- * Fetches tweets matching a query
- * @param query for which tweets need to be fetched
- * @return an array of status objects
- */
- public JSONArray GetSearchResults(String query)
- {
- try{
- //construct the request url
- String URL_PARAM_SEPERATOR = "&";
- StringBuilder url = new StringBuilder();
- url.append("https://api.twitter.com/1.1/search/tweets.json?q=");
- //query needs to be encoded
- url.append(URLEncoder.encode(query, "UTF-8"));
- url.append(URL_PARAM_SEPERATOR);
- url.append("count=100");
- URL navurl = new URL(url.toString());
- HttpURLConnection huc = (HttpURLConnection) navurl.openConnection();
- huc.setReadTimeout(5000);
- Consumer.sign(huc);
- huc.connect();
- if(huc.getResponseCode()==400||huc.getResponseCode()==404||huc.getResponseCode()==429)
- {
- System.out.println(huc.getResponseMessage());
- try {
- huc.disconnect();
- Thread.sleep(this.GetWaitTime("/friends/list"));
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
- }
- if(huc.getResponseCode()==500||huc.getResponseCode()==502||huc.getResponseCode()==503)
- {
- System.out.println(huc.getResponseMessage());
- try {
- Thread.sleep(2000);
- } catch (InterruptedException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getInputStream()));
- String temp;
- StringBuilder page = new StringBuilder();
- while( (temp = bRead.readLine())!=null)
- {
- page.append(temp);
- }
- JSONTokener jsonTokener = new JSONTokener(page.toString());
- try {
- JSONObject json = new JSONObject(jsonTokener);
- JSONArray results = json.getJSONArray("statuses");
- return results;
- } catch (JSONException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- } catch (OAuthCommunicationException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthMessageSignerException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthExpectationFailedException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- }catch(IOException ex)
- {
- ex.printStackTrace();
- }
- return null;
- }
-
- /**
- * Retrieves the rate limit status of the application
- * @return
- */
- public JSONObject GetRateLimitStatus()
- {
- try{
- URL url = new URL("https://api.twitter.com/1.1/application/rate_limit_status.json");
- HttpURLConnection huc = (HttpURLConnection) url.openConnection();
- huc.setReadTimeout(5000);
- OAuthConsumer consumer = new DefaultOAuthConsumer(utils.Configuration.CONSUMER_KEY,utils.Configuration.CONSUMER_SECRET);
- consumer.setTokenWithSecret(OAuthTokens.getAccessToken(), OAuthTokens.getAccessSecret());
- consumer.sign(huc);
- huc.connect();
- BufferedReader bRead = new BufferedReader(new InputStreamReader((InputStream) huc.getContent()));
- StringBuffer page = new StringBuffer();
- String temp= "";
- while((temp = bRead.readLine())!=null)
- {
- page.append(temp);
- }
- bRead.close();
- return (new JSONObject(page.toString()));
- } catch (JSONException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthCommunicationException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthMessageSignerException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- } catch (OAuthExpectationFailedException ex) {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }catch(IOException ex)
- {
- Logger.getLogger(RESTApiExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- return null;
- }
-
- /**
- * Initialize the file writer
- * @param path of the file
- * @param outFilename name of the file
- */
- public void InitializeWriters(String outFilename) {
- try {
- File fl = new File(outFilename);
- if(!fl.exists())
- {
- fl.createNewFile();
- }
- /**
- * Use UTF-8 encoding when saving files to avoid
- * losing Unicode characters in the data
- */
- OutFileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilename,true),"UTF-8"));
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * Close the opened filewriter to save the data
- */
- public void CleanupAfterFinish()
- {
- try {
- OutFileWriter.close();
- } catch (IOException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- /**
- * Writes the retrieved data to the output file
- * @param data containing the retrived information in JSON
- * @param user name of the user currently being written
- */
- public void WriteToFile(JSONArray searchResults)
- {
- try
- {
- for(int i=0;i<searchResults.length();i++)
- {
- try {
- OutFileWriter.write(searchResults.getJSONObject(i).toString());
- OutFileWriter.newLine();
- } catch (JSONException ex) {
- Logger.getLogger(RESTSearchExample.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * Retrieves the wait time if the API Rate Limit has been hit
- * @param api the name of the API currently being used
- * @return the number of milliseconds to wait before initiating a new request
- */
- public long GetWaitTime(String api)
- {
- JSONObject jobj = this.GetRateLimitStatus();
- if(jobj!=null)
- {
- try {
- if(!jobj.isNull("resources"))
- {
- JSONObject resourcesobj = jobj.getJSONObject("resources");
- JSONObject statusobj = resourcesobj.getJSONObject("statuses");
- JSONObject apilimit = statusobj.getJSONObject(api);
- int numremhits = apilimit.getInt("remaining");
- if(numremhits<=1)
- {
- long resettime = apilimit.getInt("reset");
- resettime = resettime*1000; //convert to milliseconds
- return resettime;
- }
- }
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- }
- return 0;
- }
-
- /**
- * Creates an OR search query from the supplied terms
- * @param queryTerms
- * @return a String formatted as term1 OR term2
- */
- public String CreateORQuery(ArrayList<String> queryTerms)
- {
- String OR_Operator = " OR ";
- StringBuffer querystr = new StringBuffer();
- int count = 1;
- for(String term:queryTerms)
- {
- if(count==1)
- {
- querystr.append(term);
- }
- else
- {
- querystr.append(OR_Operator).append(term);
- }
- }
- return querystr.toString();
- }
-
- public static void main(String[] args)
- {
- RESTSearchExample rse = new RESTSearchExample();
- ArrayList<String> queryterms = new ArrayList<String>();
- String outfilename = rse.DEF_FILENAME;
- if(args!=null)
- {
- if(args.length>0)
- {
- for(int i=0;i<args.length;i++)
- {
- queryterms.add(args[i]);
- }
- }
- else
- {
- queryterms.add(rse.query);
- }
- }
- rse.LoadTwitterToken();
- rse.Consumer = rse.GetConsumer();
- System.out.println(rse.GetRateLimitStatus());
- rse.InitializeWriters(outfilename);
- JSONArray results = rse.GetSearchResults(rse.CreateORQuery(queryterms));
- if(results!=null)
- {
- rse.WriteToFile(results);
- }
- rse.CleanupAfterFinish();
- }
-}
diff --git a/src/Chapter2/streamingapi/StreamingApiExample.java b/src/Chapter2/streamingapi/StreamingApiExample.java
deleted file mode 100644
index 61a9cc0..0000000
--- a/src/Chapter2/streamingapi/StreamingApiExample.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.streamingapi;
-
-import Chapter2.support.OAuthTokenSecret;
-import Chapter2.openauthentication.OAuthExample;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import oauth.signpost.OAuthConsumer;
-import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
-import oauth.signpost.exception.OAuthCommunicationException;
-import oauth.signpost.exception.OAuthExpectationFailedException;
-import oauth.signpost.exception.OAuthMessageSignerException;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.HttpEntity;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.params.CoreConnectionPNames;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-import utils.Configuration;
-
-public class StreamingApiExample
-{
- OAuthTokenSecret OAuthToken;
- final int RECORDS_TO_PROCESS = 1000;
- final int MAX_GEOBOXES = 25;
- final int MAX_KEYWORDS = 400;
- final int MAX_USERS = 5000;
- HashSet<String> Keywords;
- HashSet<String> Geoboxes;
- HashSet<String> Userids;
- final String CONFIG_FILE_PATH = "streaming/streaming.config";
- final String DEF_OUTPATH = "streaming/";
-
- /**
- * Loads the Twitter access token and secret for a user
- */
- public void LoadTwitterToken()
- {
-// OAuthExample oae = new OAuthExample();
-// OAuthToken = oae.GetUserAccessKeySecret();
- OAuthToken = OAuthExample.DEBUGUserAccessSecret();
- }
-
- /**
- * Creates a connection to the Streaming Filter API
- * @param baseUrl the URL for Twitter Filter API
- * @param outFilePath Location to place the exported file
- */
- public void CreateStreamingConnection(String baseUrl, String outFilePath)
- {
- HttpClient httpClient = new DefaultHttpClient();
- httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, new Integer(90000));
- //Step 1: Initialize OAuth Consumer
- OAuthConsumer consumer = new CommonsHttpOAuthConsumer(Configuration.CONSUMER_KEY,Configuration.CONSUMER_SECRET);
- consumer.setTokenWithSecret(OAuthToken.getAccessToken(),OAuthToken.getAccessSecret());
- //Step 2: Create a new HTTP POST request and set parameters
- HttpPost httppost = new HttpPost(baseUrl);
- try {
- httppost.setEntity(new UrlEncodedFormEntity(CreateRequestBody(), "UTF-8"));
- } catch (UnsupportedEncodingException ex) {
- ex.printStackTrace();
- }
- try {
- //Step 3: Sign the request
- consumer.sign(httppost);
- } catch (OAuthMessageSignerException ex) {
- ex.printStackTrace();
- } catch (OAuthExpectationFailedException ex) {
- ex.printStackTrace();
- } catch (OAuthCommunicationException ex) {
- ex.printStackTrace();
- }
- HttpResponse response;
- InputStream is = null;
- try {
- //Step 4: Connect to the API
- response = httpClient.execute(httppost);
- if (response.getStatusLine().getStatusCode()!= HttpStatus.SC_OK)
- {
- throw new IOException("Got status " +response.getStatusLine().getStatusCode());
- }
- else
- {
- System.out.println(OAuthToken.getAccessToken()+ ": Processing from " + baseUrl);
- HttpEntity entity = response.getEntity();
- try {
- is = entity.getContent();
- } catch (IOException ex) {
- ex.printStackTrace();
- } catch (IllegalStateException ex) {
- ex.printStackTrace();
- }
- //Step 5: Process the incoming Tweet Stream
- this.ProcessTwitterStream(is, outFilePath);
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }finally {
- // Abort the method, otherwise releaseConnection() will
- // attempt to finish reading the never-ending response.
- // These methods do not throw exceptions.
- if(is!=null)
- {
- try {
- is.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
- }
- }
-
- /**
- * Processes a stream of tweets and writes them to a file one tweet per line. Each tweet here is represented by a JSON document.
- * @param is input stream already connected to the streaming API
- * @param outFilePath file to put the collected tweets in
- * @throws InterruptedException
- * @throws IOException
- */
- public void ProcessTwitterStream(InputStream is, String outFilePath)
- {
- BufferedWriter bwrite = null;
- try {
- JSONTokener jsonTokener = new JSONTokener(new InputStreamReader(is, "UTF-8"));
- ArrayList<JSONObject> rawtweets = new ArrayList<JSONObject>();
- int nooftweetsuploaded = 0;
- while (true) {
- try {
- JSONObject temp = new JSONObject(jsonTokener);
- rawtweets.add(temp);
-// System.out.println(temp);
- if (rawtweets.size() >= RECORDS_TO_PROCESS)
- {
- Calendar cal = Calendar.getInstance();
- String filename = outFilePath + "tweets_" + cal.getTimeInMillis() + ".json";
- bwrite = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), "UTF-8"));
- nooftweetsuploaded += RECORDS_TO_PROCESS;
- //Write the collected tweets to a file
- for (JSONObject jobj : rawtweets) {
- bwrite.write(jobj.toString());
- bwrite.newLine();
- }
- System.out.println("Written "+nooftweetsuploaded+" records so far");
- bwrite.close();
- rawtweets.clear();
- }
- } catch (JSONException ex) {
- ex.printStackTrace();
- }
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- public static void main(String[] args)
- {
- StreamingApiExample sae = new StreamingApiExample();
- sae.LoadTwitterToken();
- //load parameters from a TSV file
- String filename = sae.CONFIG_FILE_PATH;
- String outfilepath = sae.DEF_OUTPATH;
- if(args!=null)
- {
- if(args.length>0)
- {
- filename = args[0];
- }
- if(args.length>1)
- {
- File fl = new File(args[1]);
- if(fl.exists()&&fl.isDirectory())
- {
- outfilepath = args[1];
- }
- }
- }
- sae.ReadParameters(filename);
- sae.CreateStreamingConnection("https://stream.twitter.com/1.1/statuses/filter.json", outfilepath);
- }
-
- /**
- * Reads the file and loads the parameters to be crawled. Expects that the parameters are tab separated values and the
- * @param filename
- */
- public void ReadParameters(String filename)
- {
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8"));
- String temp = "";
- int count = 1;
- if(Userids==null)
- {
- Userids = new HashSet<String>();
- }
- if(Geoboxes==null)
- {
- Geoboxes = new HashSet<String>();
- }
- if(Keywords==null)
- {
- Keywords = new HashSet<String>();
- }
- while((temp = br.readLine())!=null)
- {
- if(!temp.isEmpty())
- {
- if(count==1)
- {
- String[] keywords = temp.split("\t");
- HashSet<String> temptags = new HashSet<String>();
- for(String word:keywords)
- {
- if(!temptags.contains(word))
- {
- temptags.add(word);
- }
- }
- FilterKeywords(temptags);
- }
- else
- if(count==2)
- {
- String[] geoboxes = temp.split("\t");
- HashSet<String> tempboxes = new HashSet<String>();
- for(String box:geoboxes)
- {
- if(!tempboxes.contains(box))
- {
- tempboxes.add(box);
- }
- }
- FilterGeoboxes(tempboxes);
- }
- else
- if(count==3)
- {
- String[] userids = temp.split("\t");
- HashSet<String> tempids = new HashSet<String>();
- for(String id:userids)
- {
- if(!tempids.contains(id))
- {
- tempids.add(id);
- }
- }
- FilterUserids(tempids);
- }
- count++;
- }
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- finally{
- try {
- br.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
- }
-
- private void FilterUserids(HashSet<String> userids)
- {
- if(userids!=null)
- {
- int maxsize = MAX_USERS;
- if(userids.size()<maxsize)
- {
- maxsize = userids.size();
- }
- for(String id:userids)
- {
- Userids.add(id);
- }
- }
- }
-
- private void FilterGeoboxes(HashSet<String> geoboxes)
- {
- if(geoboxes!=null)
- {
- int maxsize = MAX_GEOBOXES;
- if(geoboxes.size()<maxsize)
- {
- maxsize = geoboxes.size();
- }
- for(String box:geoboxes)
- {
- Geoboxes.add(box);
- }
- }
- }
- /**
- * Keep only the maximum permitted number of parameters for a connection. Ignoring the rest.
- * This can be extended to create multiple sets to be crawled by different threads.
- */
- private void FilterKeywords(HashSet<String> hashtags)
- {
- if(hashtags!=null)
- {
- int maxsize = MAX_KEYWORDS;
- if(hashtags.size()<maxsize)
- {
- maxsize = hashtags.size();
- }
- for(String tag:hashtags)
- {
- Keywords.add(tag);
- }
- }
-
- }
-
- private List<NameValuePair> CreateRequestBody()
- {
- List<NameValuePair> params = new ArrayList<NameValuePair>();
- if(Userids != null&&Userids.size()>0)
- {
- params.add(CreateNameValuePair("follow", Userids));
- System.out.println("userids = "+Userids);
- }
- if (Geoboxes != null&&Geoboxes.size()>0) {
- params.add(CreateNameValuePair("locations", Geoboxes));
- System.out.println("locations = "+Geoboxes);
-
- }
- if (Keywords != null&&Keywords.size()>0) {
- params.add(CreateNameValuePair("track", Keywords));
- System.out.println("keywords = "+Keywords);
- }
- return params;
- }
-
- private NameValuePair CreateNameValuePair(String name, Collection<String> items)
- {
- StringBuilder sb = new StringBuilder();
- boolean needComma = false;
- for (String item : items) {
- if (needComma) {
- sb.append(',');
- }
- needComma = true;
- sb.append(item);
- }
- return new BasicNameValuePair(name, sb.toString());
- }
-}
diff --git a/src/Chapter2/support/APIType.java b/src/Chapter2/support/APIType.java
deleted file mode 100644
index 94449f8..0000000
--- a/src/Chapter2/support/APIType.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.support;
-
-public class APIType
-{
- public static String USER_TIMELINE = "/statuses/user_timeline";
- public static String FOLLOWERS = "/followers/list";
- public static String FRIENDS = "/friends/list";
- public static String USER_PROFILE = "/users/show";
-}
diff --git a/src/Chapter2/support/InfoType.java b/src/Chapter2/support/InfoType.java
deleted file mode 100644
index 42b0334..0000000
--- a/src/Chapter2/support/InfoType.java
+++ /dev/null
@@ -1,12 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.support;
-
-public class InfoType
-{
- public static final int PROFILE_INFO = 0;
- public static final int FOLLOWER_INFO = 1;
- public static final int FRIEND_INFO = 2;
- public static final int STATUSES_INFO = 3;
-}
diff --git a/src/Chapter2/support/Location.java b/src/Chapter2/support/Location.java
deleted file mode 100644
index 7f6234f..0000000
--- a/src/Chapter2/support/Location.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * To change this template, choose Tools | Templates
- * and open the template in the editor.
- */
-
-package Chapter2.support;
-
-/**
- *
- * @author shamanth
- */
-public class Location
-{
- public Double latitude;
- public Double longitude;
-
- public Location(Double lat,Double lng)
- {
- latitude = lat;
- longitude = lng;
- }
-
- @Override
- public String toString()
- {
- return "Latitude: "+latitude+" & Longitude: "+longitude;
- }
-}
diff --git a/src/Chapter2/support/OAuthTokenSecret.java b/src/Chapter2/support/OAuthTokenSecret.java
deleted file mode 100644
index 8fee4a8..0000000
--- a/src/Chapter2/support/OAuthTokenSecret.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/* TweetTracker. Copyright (c) Arizona Board of Regents on behalf of Arizona State University
- * @author shamanth
- */
-package Chapter2.support;
-
-public class OAuthTokenSecret
-{
- String UserAccessToken;
- String UserAccessSecret;
-
- public String getAccessSecret() {
- return UserAccessSecret;
- }
-
- public void setAccessSecret(String AccessSecret) {
- this.UserAccessSecret = AccessSecret;
- }
-
- public String getAccessToken() {
- return UserAccessToken;
- }
-
- public void setAccessToken(String AccessToken) {
- this.UserAccessToken = AccessToken;
- }
-
- public OAuthTokenSecret(String token,String secret)
- {
- this.setAccessToken(token);
- this.setAccessSecret(secret);
- }
-
- @Override
- public String toString()
- {
- return "Access Token: "+getAccessToken()+" Access Secret: "+getAccessSecret();
- }
-}
generated by cgit v1.2.1 (git 2.18.0) at 2024-05-19 09:02:24 +0000