summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaurice Laveaux <m.laveaux@student.tue.nl>2014-05-22 16:36:33 +0200
committerMaurice Laveaux <m.laveaux@student.tue.nl>2014-05-22 16:36:33 +0200
commit984f0eaf7d7ce1cabdee883fd28694483bda6d85 (patch)
treec51959a87fe8aa86a8f976cdf6048bc50fd72e39
parent4f32eedd2bd49837cc297acce399c108e8b558a7 (diff)
downloadTwitterDataAnalytics-984f0eaf7d7ce1cabdee883fd28694483bda6d85.tar.gz
Added saving and loaded the search cache.
* TODO: threads don't exit safe, so cache is not valid.
-rw-r--r--src/io/SearchImpl.java68
1 files changed, 56 insertions, 12 deletions
diff --git a/src/io/SearchImpl.java b/src/io/SearchImpl.java
index c08675b..82925cf 100644
--- a/src/io/SearchImpl.java
+++ b/src/io/SearchImpl.java
@@ -4,8 +4,15 @@ import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.Writer;
import static java.lang.Thread.sleep;
+import java.util.Scanner;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
@@ -52,10 +59,8 @@ public class SearchImpl implements Search {
public SearchImpl(TwitterApi oauth) {
this.oauth = oauth;
-
requestQueue = new LinkedBlockingQueue();
-
- //load("queue_cache");
+ load("queue_cache");
}
@Override
@@ -81,13 +86,31 @@ public class SearchImpl implements Search {
}
@Override
- public void save(String filename) {
-
+ public final void save(String filename) {
+ try {
+ FileWriter writer = new FileWriter(filename);
+
+ for (Request req : requestQueue) {
+ writer.append(req.toString() + "\n");
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(SearchImpl.class.getName()).log(Level.SEVERE, null, ex);
+ }
}
@Override
- public void load(String filename) {
-
+ public final void load(String filename) {
+ try {
+ FileInputStream inputFile = new FileInputStream(filename);
+ Scanner input = new Scanner(inputFile);
+
+ while (input.hasNext()) {
+ String line = input.nextLine();
+ requestQueue.offer(new Request(line));
+ }
+ } catch (FileNotFoundException ex) {
+ save(filename);
+ }
}
@Override
@@ -98,7 +121,8 @@ public class SearchImpl implements Search {
@Override
public void close() {
if (thread_cached != null) {
- thread_cached.stop();
+ thread_cached.stop();
+ save("search_cache.txt");
}
}
@@ -111,6 +135,20 @@ public class SearchImpl implements Search {
sinceId = 0;
maxId = Long.MAX_VALUE;
}
+
+ public Request(String parse) {
+ String[] args = parse.split(":", 4);
+
+ this.screenName = args[0];
+ this.sinceId = Integer.parseInt(args[1]);
+ this.maxId = Integer.parseInt(args[2]);
+ this.numberOfTweets = Integer.parseInt(args[3]);
+ }
+
+ @Override
+ public String toString() {
+ return screenName + ":" + Long.toString(sinceId) + ":" + Long.toString(maxId) + ":" + Integer.toString(numberOfTweets);
+ }
public String screenName;
public long sinceId;
@@ -149,9 +187,10 @@ public class SearchImpl implements Search {
worker.stop();
while (workerThread.isAlive()) {
try {
+ workerThread.interrupt();
workerThread.join();
+ ioThread.interrupt();
ioThread.join();
- save("search_cached.txt");
} catch (InterruptedException ex) {
Logger.getLogger(getClass().getName())
.warning("Interrupted while waiting for search saving.");
@@ -199,10 +238,14 @@ public class SearchImpl implements Search {
public void run() {
while (true) {
if (!requestQueue.isEmpty()) {
+ Request request = null;
try {
- Request request = requestQueue.take();
+ request = requestQueue.take();
executeRequest(request);
} catch (InterruptedException ex) {
+ if (request != null) {
+ requestQueue.offer(request);
+ }
getLogger().log(Level.SEVERE, null, ex);
}
}
@@ -248,7 +291,7 @@ public class SearchImpl implements Search {
if (request.numberOfTweets > 0) {
requestQueue.offer(request);
}
-
+
if (result.getRateLimitRemaining() == 0) {
waitTil(result.getRateLimitReset());
}
@@ -295,7 +338,8 @@ public class SearchImpl implements Search {
while (worker.isRunning() || worker.hasObjects()) {
try {
processObject(worker.getObject());
- } catch (InterruptedException | ClassCastException ex) {
+ } catch (InterruptedException ex) {
+ // requested to stop.
getLogger().log(Level.SEVERE, null, ex);
}
}