Merge branch 'master' of git.lekensteyn.nl:tue/2IOC0-DBL/Goldfarmer

Conflicts:
	src/main/Analyzor.java

2 files changed, 201 insertions, 216 deletions

diff --git a/src/main/Analyzor.java b/src/main/Analyzor.java
index 2dc482b..639f385 100644
--- a/src/main/Analyzor.java
+++ b/src/main/Analyzor.java
@@ -1,211 +1,193 @@
-package main;

-

-import analysis.BrandChecker;

-import database.NamedPreparedStatement;

-import database.QueryUtils;

-import java.io.FileNotFoundException;

-import java.io.IOException;

-import java.sql.Connection;

-import java.sql.PreparedStatement;

-import java.sql.ResultSet;

-import java.sql.SQLException;

-import java.util.HashMap;

-import java.util.List;

-import java.util.Scanner;

-

-/**

- * The sentiment analysis class that rates tweets based on a unigram and bigram

- * set of weights.

- */

-public class Analyzor {

-

-    /**

-     * The map that matches single words to their weights.

-     */

-    private final HashMap<String, Double> unimap = new HashMap();

-

-    /**

-     * The map that matches word pairs to their weights.

-     */

-    private final HashMap<String, Double> bimap = new HashMap();

-

-    /**

-     * The results of a query.

-     */

-    private ResultSet data;

-    

-    /**

-     * The connection to the database.

-     */

-    private final Connection connection;

-

-    /**

-     * Used to determine the brands inside a tweet.

-     */

-    private final BrandChecker brandChecker = new BrandChecker("brands.txt");

-    

-    Analyzor(Connection connection) {

-        this.connection = connection;

-    }

-

-    //reads the lexicons

-    void readLexicon() throws FileNotFoundException {

-        if (!unimap.isEmpty()) {

-            // data is already read.

-            return;

-        }

-        // A unigram is in the format (WS = whitespace):

-        // word <WS> rating <WS> ??? <WS> ??

-        // A bigram has an two WS-separated words instead of one.

-        try (Scanner uniScanner = new Scanner("unigrams-pmilexicon.txt");

-                Scanner biScanner = new Scanner("bigrams-pmilexicon.txt");) {

-            //Fill the map of unigrams

-            while (uniScanner.hasNext()) {

-                String words = uniScanner.next();

-                unimap.put(words.toLowerCase(), uniScanner.nextDouble());

-                if (uniScanner.hasNextLine()) {

-                    uniScanner.nextLine();

-                }

-            }

-

-            //fill the map of bigrams

-            while (biScanner.hasNext()) {

-                String words = biScanner.next() + " " + biScanner.next();

-                bimap.put(words.toLowerCase(), biScanner.nextDouble());

-                if (biScanner.hasNextLine()) {

-                    biScanner.nextLine();

-                }

-            }

-        }

-    }

-

-    /**

-     * Executes a query that the analyzer can analyze.

-     *

-     * @param query The query string to execute.

-     * @throws SQLException When database connection isn't available.

-     */

-    public void query(String query) throws SQLException {

-        PreparedStatement statement;

-        //make a connection to the database and execute the query

-        statement = connection.prepareStatement(query);

-        data = statement.executeQuery();

-    }

-

-    /**

-     * Run a sentiment analysis and fill the database with the output.

-     *

-     * @param query The query to analyze

-     * @throws SQLException

-     * @throws IOException

-     */

-    public void sentimentAnalysis(String query) throws SQLException, IOException {

-        query(query);

-

-        //read the lexicons

-        readLexicon();

-

-        //go to the start of te dataset

-        if (data == null) {

-            System.err.println("data is empty, try querying first");

-            return;

-        }

-        data.beforeFirst();

-

-        Double value;

-        String text;

-

-        //for all tuples

-        while (data.next()) {

-            //get the text

-            text = data.getString("text");

-            text = splitPunctToWords(text);

-            // test is the tweet text you are going to analyze

-            String[] words = text.split("\\s+"); // text splitted into separate words

-            double positiverate = 0; // positive rating

-

-            // Rate the text with unigrams

-            for (String word : words) {

-                value = unimap.get(word);

-                if (value != null) {

-                    positiverate += unimap.get(word);

-                }

-            }

-            // Rate the text with bigrams

-            for (int i = 0; i < words.length - 1; i++) {

-                String pair = words[i] + " " + words[i + 1];

-                value = bimap.get(pair);

-                if (value != null) {

-                    positiverate += bimap.get(pair);

-                }

-            }

-            // Obtain the brands contained in a tweet text.

-            //List<String> brands = brandChecker.getBrands(text);

-            

-            // insert the rating into the database

-            NamedPreparedStatement m_insertRating;

-            m_insertRating = new NamedPreparedStatement(connection, QueryUtils.insertRating);

-            QueryUtils.setInsertParams(m_insertRating, data.getLong("tweetid"), data.getString("brand"), (int) (positiverate * 10));

-            m_insertRating.executeUpdate();

-            

-            

-            //don't print the rate

-            //System.out.println(text + ": " + (int) (positiverate * 10));

-        }

-    }

-

-    //makes a wordcloud of the tweets in the ResultSet data

-    public void makeWordCloud(String query) throws SQLException {

-

-        query(query);

-        //go to the start of the ResultSet data

-        if (data == null) {

-            System.err.println("data is empty, try querying first");

-            return;

-        }

-

-        //make the hashmap with the words and their frequency

-        HashMap<String, Integer> wordcloud = new HashMap<>();

-

-        String text;

-        String[] words;

-        Integer value;

-

-        while (data.next()) {

-            //get the text

-            text = data.getString("text");

-            //remove punctuation, convert to lowercase and split on words

-            text = removePunct(text);

-            text = text.toLowerCase();

-            words = text.split("\\s+");

-

-            //count the words

-            for (String word : words) {

-                value = wordcloud.get(word);

-                if (value == null) {

-                    wordcloud.put(word, 1);

-                } else {

-                    wordcloud.put(word, value++);

-                }

-            }

-        }

-    }

-

-    //replaces punctuation so it will be splitted

-    //also removes urls

-    private String splitPunctToWords(String text) {

-        text = text.replaceAll("https?://\\S*", "");

-        text = text.replaceAll("[!?):;\"']", " $0");

-        text = text.replaceAll("[.,-](\\s|$)", " $0");

-        text = text.replaceAll("\\s[(\"']", "$0 ");

-        return text;

-    }

-

-    //removes punctuation

-    //also removes urls

-    private String removePunct(String text) {

-        text = text.replaceAll("https?://\\S*", "");

-        text = text.replaceAll("[.,!?()-:;\"']", " ");

-        return text;

-    }

-}

+package main;
+
+import database.NamedPreparedStatement;
+import database.QueryUtils;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Scanner;
+
+/**
+ * The sentiment analysis class that rates tweets based on a unigram and bigram
+ * set of weights.
+ */
+public class Analyzor {
+
+    /**
+     * The map that matches single words to their weights.
+     */
+    private final HashMap<String, Double> unimap = new HashMap();
+
+    /**
+     * The map that matches word pairs to their weights.
+     */
+    private final HashMap<String, Double> bimap = new HashMap();
+
+    private ResultSet data;
+    private final Connection connection;
+
+    Analyzor(Connection connection) {
+        this.connection = connection;
+    }
+
+    //reads the lexicons
+    void readLexicon() throws FileNotFoundException {
+        if (!unimap.isEmpty()) {
+            // data is already read.
+            return;
+        }
+        System.err.println("Trying to read lexicons...");
+        // A unigram is in the format (WS = whitespace):
+        // word <WS> rating <WS> ??? <WS> ??
+        // A bigram has an two WS-separated words instead of one.
+        try (Scanner uniScanner = new Scanner(new File("unigrams-pmilexicon.txt"));
+                Scanner biScanner = new Scanner(new File("bigrams-pmilexicon.txt"));) {
+            //Fill the map of unigrams
+            while (uniScanner.hasNext()) {
+                String words = uniScanner.next();
+                unimap.put(words.toLowerCase(), uniScanner.nextDouble());
+                if (uniScanner.hasNextLine()) {
+                    uniScanner.nextLine();
+                }
+            }
+
+            //fill the map of bigrams
+            while (biScanner.hasNext()) {
+                String words = biScanner.next() + " " + biScanner.next();
+                bimap.put(words.toLowerCase(), biScanner.nextDouble());
+                if (biScanner.hasNextLine()) {
+                    biScanner.nextLine();
+                }
+            }
+        }
+        System.err.println("Lexicons are read.");
+    }
+
+    /**
+     * Executes a query that the analyzer can analyze.
+     *
+     * @param query The query string to execute.
+     * @throws SQLException When database connection isn't available.
+     */
+    public void query(String query) throws SQLException {
+        PreparedStatement statement;
+        //make a connection to the database and execute the query
+        statement = connection.prepareStatement(query);
+        data = statement.executeQuery();
+    }
+
+    /**
+     * Run a sentiment analysis and fill the database with the output.
+     *
+     * @throws SQLException
+     * @throws IOException
+     */
+    public void sentimentAnalysis(String query) throws SQLException, IOException {
+        query(query);
+
+        //read the lexicons
+        readLexicon();
+
+        //go to the start of te dataset
+        if (data == null) {
+            System.err.println("data is empty, try querying first");
+            return;
+        }
+
+        Double value;
+        String text;
+
+        //for all tuples
+        while (data.next()) {
+            //get the text
+            text = data.getString("text");
+            text = splitPunctToWords(text);
+            // test is the tweet text you are going to analyze
+            String[] words = text.split("\\s+"); // text splitted into separate words
+            double positiverate = 0; // positive rating
+
+            // Rate the text with unigrams
+            for (String word : words) {
+                value = unimap.get(word);
+                if (value != null) {
+                    positiverate += unimap.get(word);
+                }
+            }
+            // Rate the text with bigrams
+            for (int i = 0; i < words.length - 1; i++) {
+                String pair = words[i] + " " + words[i + 1];
+                value = bimap.get(pair);
+                if (value != null) {
+                    positiverate += bimap.get(pair);
+                }
+            }
+            //insert the rating into the database
+            NamedPreparedStatement m_insertRating;
+            m_insertRating = new NamedPreparedStatement(connection, QueryUtils.insertRating);
+            QueryUtils.setInsertParams(m_insertRating, data.getLong("tweetid"), data.getString("brand"), (int) (positiverate * 10));
+            m_insertRating.executeUpdate();
+            //don't print the rate
+            //System.out.println(text + ": " + (int) (positiverate * 10));
+        }
+    }
+
+    //makes a wordcloud of the tweets in the ResultSet data
+    void makeWordCloud(String query) throws SQLException {
+
+        query(query);
+        //go to the start of the ResultSet data
+        if (data == null) {
+            System.err.println("data is empty, try querying first");
+            return;
+        }
+
+        //make the hashmap with the words and their frequency
+        HashMap<String, Integer> wordcloud = new HashMap<>();
+
+        String text;
+        String[] words;
+        Integer value;
+
+        while (data.next()) {
+            //get the text
+            text = data.getString("text");
+            //remove punctuation, convert to lowercase and split on words
+            text = removePunct(text);
+            text = text.toLowerCase();
+            words = text.split("\\s+");
+
+            //count the words
+            for (String word : words) {
+                value = wordcloud.get(word);
+                if (value == null) {
+                    wordcloud.put(word, 1);
+                } else {
+                    wordcloud.put(word, value++);
+                }
+            }
+        }
+    }
+
+    //replaces punctuation so it will be splitted
+    //also removes urls
+    private String splitPunctToWords(String text) {
+        text = text.replaceAll("https?://\\S*", "");
+        text = text.replaceAll("[!?):;\"']", " $0");
+        text = text.replaceAll("[.,-](\\s|$)", " $0");
+        text = text.replaceAll("\\s[(\"']", "$0 ");
+        return text;
+    }
+
+    //removes punctuation
+    //also removes urls
+    private String removePunct(String text) {
+        text = text.replaceAll("https?://\\S*", "");
+        text = text.replaceAll("[.,!?()-:;\"']", " ");
+        return text;
+    }
+}
diff --git a/src/main/FarmShell.java b/src/main/FarmShell.java
index 3c6f17f..7186c2f 100644
--- a/src/main/FarmShell.java
+++ b/src/main/FarmShell.java
@@ -64,8 +64,6 @@ public class FarmShell {
             // print prompt for reading next line
             printPrompt();
         }
-        // prevent corrupted compressed files when exiting without a command
-        throw new NoSuchElementException();
     }
 
     /**
@@ -102,8 +100,13 @@ public class FarmShell {
             System.err.println("Command " + args[0] + " failed with " + ex);
             ex.printStackTrace();
         } catch (NoSuchElementException ex) {
-            // thrown by the "exit" command to signal exit
-            return false;
+            if ("EXIT NOW".equals(ex.getMessage())) {
+                // thrown by the "exit" command to signal exit
+                return false;
+            } else {
+                System.err.println("ZOMG SOMETHIGN FAILED: " + ex.getMessage());
+                ex.printStackTrace();
+            }
         } catch (SQLException ex) {
             System.err.println("such " + ex);
         }
@@ -145,7 +148,7 @@ public class FarmShell {
                 }
                 break;
             case exit:
-                throw new NoSuchElementException();
+                throw new NoSuchElementException("EXIT NOW");
             default:
                 throw new AssertionError(command.name());
         }