diff options
author | Peter Wu <peter@lekensteyn.nl> | 2014-06-04 17:13:19 +0200 |
---|---|---|
committer | Peter Wu <peter@lekensteyn.nl> | 2014-06-04 17:13:19 +0200 |
commit | e14c671f3cbb56a08d765bd992e4cf774a0d1353 (patch) | |
tree | 48294b4824105f2500a3529e10cc9272505bd428 /src/main/Analyzor.java | |
parent | 6ed880b928ceaee3935562c2eb975ddaa49a8530 (diff) | |
download | Goldfarmer-e14c671f3cbb56a08d765bd992e4cf774a0d1353.tar.gz |
Producer / consumer for getBrands
Diffstat (limited to 'src/main/Analyzor.java')
-rw-r--r-- | src/main/Analyzor.java | 31 |
1 files changed, 14 insertions, 17 deletions
diff --git a/src/main/Analyzor.java b/src/main/Analyzor.java index 5a201be..810fc4d 100644 --- a/src/main/Analyzor.java +++ b/src/main/Analyzor.java @@ -1,6 +1,7 @@ package main; import analysis.BrandChecker; +import database.BrandAnalyzerQueue; import database.NamedPreparedStatement; import database.QueryUtils; import java.io.File; @@ -255,8 +256,6 @@ public class Analyzor { * @throws SQLException If the query is unsuccesfull. */ public void getBrands(String queryText, boolean reset) throws SQLException { - BrandChecker checker = new BrandChecker("brandonlyrules.txt"); - PreparedStatement statement; // make a connection to the database and execute the query if (reset) { @@ -278,29 +277,27 @@ public class Analyzor { int brandCount = 0; int count = 0; long timestamp = System.currentTimeMillis(); - - while (data.next()) { - List<String> brands = checker.getBrands(data.getString("text")); - if (brands.isEmpty()) { - brandCount++; - QueryUtils.setInsertBrandParams(insertBrand, data.getLong("tweetid"), "no"); + BrandAnalyzerQueue analyzer = new BrandAnalyzerQueue(data); + BrandAnalyzerQueue.Result result; + new Thread(analyzer).start(); + while ((result = analyzer.next()) != null) { + for (String brand : result.brands) { + QueryUtils.setInsertBrandParams(insertBrand, result.tweetid, brand); insertBrand.executeUpdate(); - } else { - brandCount += brands.size(); - for (String brand : brands) { - QueryUtils.setInsertBrandParams(insertBrand, data.getLong("tweetid"), brand); - insertBrand.executeUpdate(); - } } + brandCount += result.brands.size(); count++; if (count % 10000 == 0) { - System.out.println("Processed " + count + " tweets in " + (System.currentTimeMillis() - timestamp) + " ms"); + System.err.println("Processed " + count + " tweets, inserted " + + brandCount + " in " + ((System.currentTimeMillis() - timestamp) / 1000) + " sec"); } } - System.out.println("Processed " + count + " tweets in " + (System.currentTimeMillis() - timestamp) + " ms"); - System.out.println("Finished getBrands, processed " + count + " number of tweets, added " + brandCount + " brands or no."); + System.err.println("Processed " + count + " tweets in " + + ((System.currentTimeMillis() - timestamp) / 1000) + " sec"); + System.err.println("Finished getBrands, processed " + count + + " number of tweets, added " + brandCount + " brands or no."); } //gets the amount of users that tweet about a brand in a timezone |