diff options
| -rw-r--r-- | nbproject/build-impl.xml | 18 | ||||
| -rw-r--r-- | nbproject/genfiles.properties | 4 | ||||
| -rw-r--r-- | nbproject/project.properties | 4 | ||||
| -rw-r--r-- | src/database/DBConnection.java | 52 | ||||
| -rw-r--r-- | src/database/DBQuery.java | 20 | ||||
| -rw-r--r-- | src/database/Post.java (renamed from src/main/Post.java) | 126 | ||||
| -rw-r--r-- | src/database/QueryUtils.java | 40 | ||||
| -rw-r--r-- | src/main/Main.java | 114 |
8 files changed, 267 insertions, 111 deletions
diff --git a/nbproject/build-impl.xml b/nbproject/build-impl.xml index ef573dd..d88ddb4 100644 --- a/nbproject/build-impl.xml +++ b/nbproject/build-impl.xml @@ -80,9 +80,12 @@ is divided into following sections: </and> </condition> <condition property="do.archive"> - <not> - <istrue value="${jar.archive.disabled}"/> - </not> + <or> + <not> + <istrue value="${jar.archive.disabled}"/> + </not> + <istrue value="${not.archive.disabled}"/> + </or> </condition> <condition property="do.mkdist"> <and> @@ -1199,11 +1202,14 @@ is divided into following sections: </not> </and> </condition> - <javadoc additionalparam="${javadoc.additionalparam}" author="${javadoc.author}" charset="UTF-8" destdir="${dist.javadoc.dir}" docencoding="UTF-8" encoding="${javadoc.encoding.used}" failonerror="true" noindex="${javadoc.noindex}" nonavbar="${javadoc.nonavbar}" notree="${javadoc.notree}" private="${javadoc.private}" source="${javac.source}" splitindex="${javadoc.splitindex}" use="${javadoc.use}" useexternalfile="true" version="${javadoc.version}" windowtitle="${javadoc.windowtitle}"> + <condition else="" property="bug5101868workaround" value="*.java"> + <matches pattern="1\.[56](\..*)?" string="${java.version}"/> + </condition> + <javadoc additionalparam="-J-Dfile.encoding=${file.encoding} ${javadoc.additionalparam}" author="${javadoc.author}" charset="UTF-8" destdir="${dist.javadoc.dir}" docencoding="UTF-8" encoding="${javadoc.encoding.used}" failonerror="true" noindex="${javadoc.noindex}" nonavbar="${javadoc.nonavbar}" notree="${javadoc.notree}" private="${javadoc.private}" source="${javac.source}" splitindex="${javadoc.splitindex}" use="${javadoc.use}" useexternalfile="true" version="${javadoc.version}" windowtitle="${javadoc.windowtitle}"> <classpath> <path path="${javac.classpath}"/> </classpath> - <fileset dir="${src.dir}" excludes="*.java,${excludes}" includes="${includes}"> + <fileset dir="${src.dir}" excludes="${bug5101868workaround},${excludes}" includes="${includes}"> <filename name="**/*.java"/> </fileset> <fileset dir="${build.generated.sources.dir}" erroronmissingdir="false"> @@ -1277,7 +1283,7 @@ is divided into following sections: <mkdir dir="${build.test.results.dir}"/> </target> <target depends="init,compile-test,-pre-test-run" if="have.tests" name="-do-test-run"> - <j2seproject3:test testincludes="**/*Test.java"/> + <j2seproject3:test includes="${includes}" testincludes="**/*Test.java"/> </target> <target depends="init,compile-test,-pre-test-run,-do-test-run" if="have.tests" name="-post-test-run"> <fail if="tests.failed" unless="ignore.failing.tests">Some tests failed; see details above.</fail> diff --git a/nbproject/genfiles.properties b/nbproject/genfiles.properties index e5a905b..229557a 100644 --- a/nbproject/genfiles.properties +++ b/nbproject/genfiles.properties @@ -4,5 +4,5 @@ build.xml.stylesheet.CRC32=8064a381@1.68.1.46 # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. nbproject/build-impl.xml.data.CRC32=bae11919 -nbproject/build-impl.xml.script.CRC32=220d582d -nbproject/build-impl.xml.stylesheet.CRC32=5a01deb7@1.68.1.46 +nbproject/build-impl.xml.script.CRC32=e529da7b +nbproject/build-impl.xml.stylesheet.CRC32=876e7a8f@1.74.2.48 diff --git a/nbproject/project.properties b/nbproject/project.properties index 706ba09..a7c47fc 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -34,8 +34,8 @@ file.reference.postgresql-9.3-1101.jdbc41.jar=D:\\Downloads\\postgresql-9.3-1101 includes=** jar.compress=false javac.classpath=\ - ${file.reference.postgresql-9.3-1101.jdbc41.jar}:\ - ${file.reference.json.jar} + ${file.reference.json.jar}:\ + ${file.reference.postgresql-9.3-1101.jdbc41.jar} # Space-separated list of extra javac options javac.compilerargs= javac.deprecation=false diff --git a/src/database/DBConnection.java b/src/database/DBConnection.java new file mode 100644 index 0000000..4227fb3 --- /dev/null +++ b/src/database/DBConnection.java @@ -0,0 +1,52 @@ + +package database; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Create a persistent database connection. + * @author Maurice Laveaux + */ +public class DBConnection { + + /* The interface to the postgresql database connection. */ + private Connection m_connection; + + public DBConnection(final String hostaddress, + final String port, + final String databasename, + final String username, + final String password) { + + String url = "jdbc:postgresql://" + hostaddress + ":" + port + "/" + databasename; + + try { + m_connection = DriverManager.getConnection(url, username, password); + } catch (SQLException ex) { + //TODO: retry when db connection fails or something. + throw new RuntimeException("cannot connect to host: " + url); + } + } + + public void prepare(DBQuery query) { + + } + + /** + * Closes the connection if it exists. + */ + public void close() { + if (m_connection != null) { + try { + m_connection.close(); + } catch (SQLException ex) { + /* TODO: what to do here else. */ + Logger.getLogger(DBConnection.class.getName()).log(Level.SEVERE, null, ex); + } + } + } +} diff --git a/src/database/DBQuery.java b/src/database/DBQuery.java new file mode 100644 index 0000000..602af97 --- /dev/null +++ b/src/database/DBQuery.java @@ -0,0 +1,20 @@ +package database; + +import java.sql.PreparedStatement; + +/** + * An interface to known query objects. + * @author Maurice Laveaux + */ +public class DBQuery { + + protected PreparedStatement m_prepared; + + public DBQuery(final String query) { + + } + + public void execute() { + + } +} diff --git a/src/main/Post.java b/src/database/Post.java index f3597c0..03ca11e 100644 --- a/src/main/Post.java +++ b/src/database/Post.java @@ -1,10 +1,11 @@ -package main;
+package database;
+
+
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
-
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -23,39 +24,41 @@ import org.json.JSONObject; *
* @author s129778
*/
-public class Post implements Runnable{
- Thread runner;
- JSONObject tweet;
- JSONObject twuser;
-
-
- Connection con = null;
- Statement st = null;
- ResultSet rs = null;
- PreparedStatement pst = null;
- String url = "jdbc:postgresql://131.155.240.72:5432/Twitter";
- String user = "postgres";
- String password = "2IOC02";
- public Post() {
- }
- public void start(){
- runner.start();
- }
- public Post(JSONObject tweet, JSONObject user) {
+public class Post implements Runnable {
+
+ Thread runner;
+ JSONObject tweet;
+ JSONObject twuser;
+
+ Connection con = null;
+ Statement st = null;
+ ResultSet rs = null;
+ PreparedStatement pst = null;
+ String url = "jdbc:postgresql://131.155.240.72:5432/Twitter";
+ String user = "postgres";
+ String password = "2IOC02";
+
+ public Post() {
+ }
+
+ public void start() {
+ runner.start();
+ }
+
+ public Post(JSONObject tweet, JSONObject user) {
//create new tread
-
- runner = new Thread(this, "post");
- this.twuser = user;
- this.tweet = tweet;
- }
-
-
- public void run() {
+
+ runner = new Thread(this, "post");
+ this.twuser = user;
+ this.tweet = tweet;
+ }
+
+ public void run() {
try {
con = DriverManager.getConnection(url, user, password);
st = con.createStatement();
rs = st.executeQuery("SELECT VERSION()");
-
+
//st.executeUpdate("CREATE TABLE name (ID INT)");
//st.executeUpdate("DROP TABLE name");
//st.executeQuery("SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'name' AND pid <> pg_backend_pid();");
@@ -66,12 +69,12 @@ public class Post implements Runnable{ pst = con.prepareStatement(query);
rs = pst.executeQuery();
//process select query
- while ( rs.next() ) {
+ while (rs.next()) {
int id = rs.getInt("tweetid");
- System.out.println( "ID = " + id );
- System.out.println (rs.getTimestamp("createdat"));
+ System.out.println("ID = " + id);
+ System.out.println(rs.getTimestamp("createdat"));
}
-
+
//insert query with check for existance
//for(int i=0;i<900000;i++){
String update = "INSERT INTO tweet (tweetid,createdat,favcount,retweetcount,text) SELECT ? , ? ,?,?, ? WHERE NOT EXISTS (SELECT * FROM tweet WHERE tweetid= ? )";
@@ -83,16 +86,14 @@ public class Post implements Runnable{ pst.setString(5, tweet.getString("text"));
pst.setLong(6, tweet.getLong("id"));
pst.executeUpdate();
-
-
-
+
update = "INSERT INTO ispostedby (tweetid,userid) SELECT ? , ? WHERE NOT EXISTS (SELECT * FROM ispostedby WHERE tweetid= ? )";
pst = con.prepareStatement(update);
pst.setLong(1, tweet.getLong("id"));
pst.setLong(2, twuser.getLong("id"));
pst.setLong(3, tweet.getLong("id"));
pst.executeUpdate();
-
+
//update = "INSERT INTO twitteruser (userid,displayname,timezone,tweetcount,followercount,followedcount,location,brand) Select 1";
update = "INSERT INTO twitteruser (userid,displayname,timezone,tweetcount,followercount,followedcount,location) SELECT ? ,? ,? ,? , ?, ? ,? WHERE NOT EXISTS (SELECT * FROM twitteruser WHERE userid= ? )";
pst = con.prepareStatement(update);
@@ -105,9 +106,9 @@ public class Post implements Runnable{ pst.setString(7, twuser.getString("location"));
pst.setLong(8, twuser.getLong("id"));
pst.executeUpdate();
-
- ArrayList<String> brands=getBrands();
- for(String brand : brands){
+
+ ArrayList<String> brands = getBrands();
+ for (String brand : brands) {
update = "INSERT INTO likesbrand (userid,brand) SELECT ? , ? WHERE NOT EXISTS (SELECT * FROM likesbrand WHERE userid= ? AND brand = ?)";
pst = con.prepareStatement(update);
@@ -121,22 +122,14 @@ public class Post implements Runnable{ //update
//pst = con.prepareStatement("UPDATE tweet set text = 'j' where tweetid=1;");
//pst.executeUpdate();
-
+
//delete
//pst = con.prepareStatement("DELETE from tweet where tweetid=1;");
//pst.executeUpdate();
-
-
-
-
-
-
-
-
} catch (JSONException ex) {
Logger.getLogger(Post.class.getName()).log(Level.SEVERE, null, ex);
} catch (SQLException ex) {
- Logger lgr = Logger.getLogger(Main.class.getName());
+ Logger lgr = Logger.getLogger(Post.class.getName());
lgr.log(Level.SEVERE, ex.getMessage(), ex);
} finally {
@@ -152,30 +145,29 @@ public class Post implements Runnable{ }
} catch (SQLException ex) {
- Logger lgr = Logger.getLogger(Main.class.getName());
+ Logger lgr = Logger.getLogger(Post.class.getName());
lgr.log(Level.WARNING, ex.getMessage(), ex);
}
}
-
-
- }
-
- ArrayList<String> getBrands(){
- ArrayList<String> result= new ArrayList<String>();
- String text=null;
+
+ }
+
+ ArrayList<String> getBrands() {
+ ArrayList<String> result = new ArrayList<String>();
+ String text = null;
try {
text = tweet.getString("text");
text = text.toLowerCase();
} catch (JSONException ex) {
Logger.getLogger(Post.class.getName()).log(Level.SEVERE, null, ex);
}
- if(text.contains("samsung")||text.contains("galaxy")){
- result.add("Samsung");
- System.out.println("samsung");
- } else {
- result.add("geen");
- System.out.println(text);
- }
- return result;
+ if (text.contains("samsung") || text.contains("galaxy")) {
+ result.add("Samsung");
+ System.out.println("samsung");
+ } else {
+ result.add("geen");
+ System.out.println(text);
}
+ return result;
+ }
}
diff --git a/src/database/QueryUtils.java b/src/database/QueryUtils.java new file mode 100644 index 0000000..a120d1c --- /dev/null +++ b/src/database/QueryUtils.java @@ -0,0 +1,40 @@ +package database; + +import org.json.JSONObject; + +/** + * Utilities to create queries. + * + * @author Maurice Laveaux + */ +public class QueryUtils { + + /** + * Create an insert tweet. + * + * @return A valid database query. + */ + public static DBQuery insertProfile() { + String query = "INSERT INTO twitteruser (userid,displayname,timezone,tweetcount," + + "followercount,followedcount,location) " + + "SELECT ?, ?, ?, ?, ?, ?, ? " + + "WHERE NOT EXISTS " + + "(SELECT * FROM twitteruser WHERE userid= ? )"; + return new DBQuery(query); + } + + public static DBQuery insertTweet() { + String query = "INSERT INTO tweet (tweetid,createdat,favcount,retweetcount,text) " + + "SELECT ?, ?, ?, ?, ? WHERE NOT EXISTS (SELECT * FROM tweet WHERE tweetid= ? )"; + return new DBQuery(query); + } + + public static void setInsertParams(DBQuery query, JSONObject tweet) { + /*query.getPrepared().setLong(1, tweet.getLong("id")); + pst.setString(2, tweet.getString("created_at")); + pst.setLong(3, tweet.getLong("favorite_count")); + pst.setLong(4, tweet.getLong("retweet_count")); + pst.setString(5, tweet.getString("text")); + pst.setLong(6, tweet.getLong("id"));*/ + } +} diff --git a/src/main/Main.java b/src/main/Main.java index 4d11951..dec057a 100644 --- a/src/main/Main.java +++ b/src/main/Main.java @@ -1,47 +1,93 @@ package main; -import java.io.File; -import java.io.FileNotFoundException; -import java.util.Scanner; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.json.JSONObject; + +import database.DBConnection; +import io.DataReader; +import io.InputReader; +import java.util.Arrays; + /** - * - * @author maurice + * The main class. */ public class Main { - + /** - * - * @param args + * The main method of the application. + * @param args the global arguments to pass to the program. */ public static void main(String[] args) { - // TODO code application logic here - - Scanner scanner=null; try { - scanner = new Scanner(new File("tweets-en.txt")); - } catch (FileNotFoundException ex) { - Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); - } - try { - while(scanner.hasNextLine()){ - - - JSONObject tweet = new JSONObject(scanner.nextLine()); - JSONObject user = (JSONObject) tweet.get("user"); - Post p = new Post(tweet,user); - p.start(); - } - - - - } catch (Exception e){ - e.printStackTrace(); + Main main = new Main(args); + } catch (IllegalArgumentException ex) { + System.err.println(ex.getMessage()); + System.exit(1); } - - + } + + private String m_hostaddress; + + private String m_filename; + + public Main(String[] args) { + /* parse the global options. */ + parseGlobalOptions(args); + + if (m_hostaddress == null) { + throw new IllegalArgumentException("Missing --ip to specify the hostaddress."); + } + DBConnection connection = new DBConnection(m_hostaddress, "5432", "Twitter", "postgres", "2IOC02"); + /* create the object that fills the database */ + DataFiller filler = new DataFiller(connection); + + if (m_filename == null) { + InputReader reader = new InputReader(filler); + + reader.startLoop(); + } else { + DataReader reader = new DataReader(m_filename, filler); + } + connection.close(); } + + private void parseGlobalOptions(String[] args) { + /* parse global options */ + for (int i = 0; i < args.length; i++) { + if ("--help".equals(args[i])) { + printHelp(); + } else if("--dbhost".equals(args[i])) { + m_hostaddress = getParam(args, ++i); + } else if (args[i].startsWith("-")) { + throw new IllegalArgumentException("Invalid option: " + args[i]); + } else { + /* This should be the filename */ + m_filename = getParam(args, i); + } + } + } + + /** Read an extra option for a command. */ + private String getParam(String[] args, Integer index) { + if (index + 1 <= args.length) { + index++; + return args[index - 1]; + } else { + throw new IllegalArgumentException("An extra option was missing."); + } + } + + private void printHelp() { + for (String line : HELP) { + System.out.println(line); + } + } + + private final static String[] HELP = { + "Global options:", + " --help Print this help text.", + " --ip <ip> Specify the database ip address.", + " --file <tweet> <profile> Specify the tweet and profile filenames ", + " ", + " if no --file was specified read from standard input." + }; } |