最近花时间学习了一下使用Java获取网站数据的方法，自己也亲自动手实践一下；共获取3000+数据，去除重复的数据剩余2000+，使用JFreeChart根据电影评分做出几张简单的统计图。
电影评分统计图：
JFreeChart生成图片
使用jsoup获取该网站的电影数据信息，此网站动态加载数据，如果直接查看网页源代码是看不到数据的。可以通过js文件，获取相应的数据：
部分代码如下：
movieServlet.java
主要的功能为：获取网站的电影数据
首先获取每一个电影分类的链接：
1 HashMap<String, String> urlandnames = new HashMap<String, String>(); 2 MovieService movieService = new MovieService(); 3 // 排行榜页面 4 String url = "http://movie.douban.com/chart"; 5 // 获取分类的所有相对链接和分类名称 6 try { 7 Document kinds = Jsoup.connect(url) 8 .userAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36") 9 .timeout(10000) 10 .get(); 11 Elements elements = kinds.select("#content .types a"); 12 for(Element element : elements){ 13 String kindurl = element.attr("href"); // 链接地址 14 String name = element.text(); // 类别 15 urlandnames.put(kindurl,name); 16 } 17 } catch (IOException e) { 18 e.printStackTrace(); 19 System.out.println("获取urlandname出现错误！！"); 20 } 21 //获取所有的key 22 Set<String> keySet = urlandnames.keySet(); 23 //迭代key值 24 Iterator<String> iterator = keySet.iterator(); 25 List<Movie> allMovies = new ArrayList<Movie>(); 26 while(iterator.hasNext()){ 27 // 获取到key值,即url 28 String next = iterator.next(); 29 // 根据某一个类别的链接，获取行对应的电影数据 30 List<Movie> listMovie = getMovieInfo(next); 31 allMovies.addAll(listMovie); 32 }根据对应的链接获取相应的数据，保存至数据库：
1 /** 2 * 获取种类电影信息，保存到数据库 3 * @param url 某一个种类的链接地址 4 */ 5 private List<Movie> getMovieInfo(String url){ 6 String[] tempurl = url.split("&"); 7 String finalurl = "http://movie.douban.com/j/chart/top_list_count?"+tempurl[1]+"&"+tempurl[2]; 8 // finalurl ---------http://movie.douban.com/j/chart/top_list_count?type=18&interval_id=100:90 9 String document = null; 10 try { 11 //获取该类别影片的数量total、可在线观看数量playable_count 12 document = Jsoup.connect(finalurl).timeout(10000).ignoreContentType(true).execute().body(); 13 // document------{"playable_count":18,"total":32,"unwatched_count":32}可在线观看18部，共32部，未观看32部 14 } catch (IOException e) { 15 e.printStackTrace(); 16 } 17 18 //json解析器 19 JsonParser parser = new JsonParser(); 20 //获取json对象 21 JsonObject jsonObject = (JsonObject) parser.parse(document); 22 //将json数据转为int型数据 23 int movienum = jsonObject.get("total").getAsInt(); 24 System.out.println(movienum);//该类型的数量 25 String nameurl = "http://movie.douban.com/j/chart/top_list?"+tempurl[1]+"&"+tempurl[2]+"&action=&start=0&limit="+movienum; 26 // nameurl-------------http://movie.douban.com/j/chart/top_list?type=18&interval_id=100:90&action=&start=0&limit=32 27 FileWriter fw = null; 28 String doc = null; 29 try { 30 //获取该类别的所有影片的信息 31 doc = Jsoup.connect(nameurl).timeout(10000).ignoreContentType(true).execute().body(); 32 } catch (Exception e) { 33 e.printStackTrace(); 34 } 35 //将json的一个对象数组解析成JsonElement对象 36 JsonElement element = null; 37 try { 38 //通过JsonParser对象可以把json格式的字符串解析成一个JsonElement对象 39 element = parser.parse(doc); 40 } catch (NullPointerException e) { 41 e.printStackTrace(); 42 } 43 44 JsonArray jsonArray = null; 45 if(element.isJsonArray()){ 46 //JsonElement对象如果是一个数组的话转化成jsonArray 47 jsonArray = element.getAsJsonArray(); 48 } 49 50 //遍历json的对象数组 51 Iterator it = jsonArray.iterator(); 52 List<Movie> listMovie = new ArrayList<Movie>(); 53 while (it.hasNext()) { 54 JsonObject e = (JsonObject)it.next(); 55 //电影名称 56 String name = e.get("title").getAsString(); 57 //豆瓣评分 58 float score = e.get("score").getAsFloat(); 59 //发布时间 60 String release_date = e.get("release_date").getAsString(); 61 //类型 62 JsonArray jsonArray2 = e.get("types").getAsJsonArray(); 63 String types = jsonArray2.toString(); 64 //链接地址 65 String movieUrl = e.get("url").getAsString(); 66 //是否可以在线播放 67 String is_playable = e.get("is_playable").getAsString(); 68 69 String substring = movieUrl.substring(0, movieUrl.lastIndexOf("/")); 70 String keyID = substring.substring(substring.lastIndexOf("/"), substring.length()); 71 72 if(cache.get(keyID) != null){ 73 String value = (String) cache.get(keyID).getObjectValue(); 74 if(!name.equals(value)){ 75 net.sf.ehcache.Element element2 = new net.sf.ehcache.Element(keyID,name); 76 cache.put(element2); 77 }else { 78// System.out.println("重复的 movie Info"); 79 continue; 80 } 81 }else { 82 net.sf.ehcache.Element element2 = new net.sf.ehcache.Element(keyID,name); 83 cache.put(element2); 84 } 85 86 Movie movie = new Movie(); 87 88 movie.setName(name); 89 movie.setTypes(types); 90 movie.setRelease_date(release_date); 91 movie.setScore(score); 92 movie.setMovieUrl(movieUrl); 93 movie.setIs_playable(is_playable); 94 95 //在控制台输出 96// System.out.println(movie.toString()); 97// System.out.println("正在获取数据ing..."); 98 99 listMovie.add(movie); 100 } 101 return listMovie; 102 }
ScoreServlet.java 主要是生成图表
生成柱状图：
1 protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 2 String method = request.getParameter("method"); 3 System.out.println(method+"===================method"); 4 MovieService movieService = new MovieService(); 5 6 Map<String, Integer> map = movieService.Count(); 7 Integer one = map.get("one"); 8 Integer two = map.get("two"); 9 Integer three = map.get("three"); 10 Integer four = map.get("four"); 11 Integer five = map.get("five"); 12 13 if(method.equals("barChart")){ 14 double [][]data = new double[][]{{one},{two},{three},{four},{five}}; 15 String []rowKeys = {">=9",">=8.5",">=8",">=7.5","<7.5"}; 16 String []columnKeys = {"评分"}; 17 18 CategoryDataset dataset = DatasetUtilities.createCategoryDataset(rowKeys, columnKeys, data); 19 20 JFreeChart chart = ChartFactory.createBarChart3D( 21 "电影评分柱状图", // 图表标题 22 "电影", // 目录轴的显示标签 23 "数量", // 数值轴的显示标签 24 dataset, // 数据集 25 PlotOrientation.VERTICAL, // 图表方向：水平、垂直 26 true, // 是否显示图例(对于简单的柱状图必须是 false) 27 false, // 是否创建工具提示 (tooltip) 28 false // 是否生成 URL 链接 29 ); 30 31 CategoryPlot plot = chart.getCategoryPlot(); 32 // 设置网格背景颜色 33 plot.setBackgroundPaint(Color.white); 34 // 设置网格竖线颜色 35 plot.setDomainGridlinePaint(Color.pink); 36 // 设置网格横线颜色 37 plot.setRangeGridlinePaint(Color.pink); 38 39 // 显示每个柱的数值，并修改该数值的字体属性 40 BarRenderer3D renderer=new BarRenderer3D(); 41 renderer.setBaseItemLabelGenerator(new StandardCategoryItemLabelGenerator()); 42 renderer.setBaseItemLabelsVisible(true); 43 44 renderer.setBasePositiveItemLabelPosition(new ItemLabelPosition(ItemLabelAnchor.OUTSIDE12, TextAnchor.BASELINE_LEFT)); 45 renderer.setItemLabelAnchorOffset(10D); 46 47 // 设置平行柱的之间距离 48 renderer.setItemMargin(0.4); 49 plot.setRenderer(renderer); 50 51 FileOutputStream fos_jpg = null; 52 try { 53 //将图片保存至Tomcat服务器WebRoot下的img目录中 54 fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"barChart.jpg"); 55 ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 56 } catch (Exception e) { 57 System.out.println("error"); 58 } finally { 59 try { 60 fos_jpg.close(); 61 } catch (Exception e) { 62 System.out.println("error2"); 63 } 64 } 65 request.setAttribute("barChart", "barChart.jpg"); 66 67 }生成饼状图：
1 MovieService movieService = new MovieService(); 2 3 Map<String, Integer> map = movieService.Count(); 4 Integer one = map.get("one"); 5 Integer two = map.get("two"); 6 Integer three = map.get("three"); 7 Integer four = map.get("four"); 8 Integer five = map.get("five"); 9 10 if (method.equals("pieChart")) { 11 12 DefaultPieDataset data = new DefaultPieDataset(); 13 data.setValue(">=9",one); 14 data.setValue(">=8.5",two); 15 data.setValue(">=8",three); 16 data.setValue(">=7.5",four); 17 data.setValue("<7.5",five); 18 19 JFreeChart chart = ChartFactory.createPieChart3D( 20 "评分饼状图", // 图表标题 21 data, 22 true, // 是否显示图例 23 false, // 是否创建工具提示 (tooltip) 24 false // 是否生成 URL 链接 25 ); 26 27 //显示百分比 28 PiePlot pieplot = (PiePlot)chart.getPlot(); 29 pieplot.setLabelFont(new Font("宋体", 0, 12)); 30 pieplot.setNoDataMessage("无数据"); 31 pieplot.setCircular(true); 32 pieplot.setLabelGap(0.02D); 33 pieplot.setLabelGenerator(new StandardPieSectionLabelGenerator("{0} {2}",NumberFormat.getNumberInstance(),new DecimalFormat("0.00%"))); 34 35 PiePlot3D pieplot3d = (PiePlot3D)chart.getPlot(); 36 //设置开始角度 37 pieplot3d.setStartAngle(120D); 38 //设置方向为”顺时针方向“ 39 pieplot3d.setDirection(Rotation.CLOCKWISE); 40 //设置透明度，0.5F为半透明，1为不透明，0为全透明 41 pieplot3d.setForegroundAlpha(0.7F); 42 43 FileOutputStream fos_jpg = null; 44 try { 45 //将图片保存至Tomcat服务器WebRoot目录下 46 fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"pieChart.jpg"); 47 ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 48 } catch (Exception e) { 49 System.out.println("error"); 50 } finally { 51 try { 52 fos_jpg.close(); 53 } catch (Exception e) { 54 System.out.println("error2"); 55 } 56 } 57 request.setAttribute("pieChart", "pieChart.jpg"); 58 59 }生成折线图
1 if (method.equals("lineChart")) { 2 XYSeriesCollection collection = new XYSeriesCollection(); 3 XYSeries series = new XYSeries("折线"); 4 5 Map<String, Integer> map2 = movieService.lineChart(); 6 int number = 99; 7 for(int i=0; i<map2.size(); i++){ 8 String s= number+""; 9 String score = s.charAt(0)+"."+s.charAt(1); 10 series.add(Double.parseDouble(score),map2.get(score)); 11// System.out.println(Double.parseDouble(score)+"--"+map2.get(score)); 12 number--; 13 } 14 collection.addSeries(series); 15 16 JFreeChart chart = ChartFactory.createXYLineChart( 17 "评分折线图", 18 "评分", 19 "数量", 20 collection, 21 PlotOrientation.VERTICAL, 22 true, 23 true, 24 false); 25 26 XYPlot plot = (XYPlot) chart.getPlot(); 27 //设置曲线是否显示数据点 28 XYLineAndShapeRenderer xylinerenderer = (XYLineAndShapeRenderer)plot.getRenderer(); 29 xylinerenderer.setBaseShapesVisible(true); 30 31 //设置曲线显示各数据点的值 32 XYItemRenderer xyitem = plot.getRenderer(); 33 xyitem.setBaseItemLabelsVisible(true); 34 xyitem.setBasePositiveItemLabelPosition(new ItemLabelPosition(ItemLabelAnchor.OUTSIDE12, TextAnchor.BASELINE_CENTER)); 35 xyitem.setBaseItemLabelGenerator(new StandardXYItemLabelGenerator()); 36 xyitem.setBaseItemLabelFont(new Font("Dialog", 1, 10)); 37 plot.setRenderer(xyitem); 38 39 FileOutputStream fos_jpg = null; 40 try { 41 //将图片保存至Tomcat服务器WebRoot目录下 42 fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"lineChart.jpg"); 43 ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 44 } catch (Exception e) { 45 System.out.println("error"); 46 } finally { 47 try { 48 fos_jpg.close(); 49 } catch (Exception e) { 50 System.out.println("error2"); 51 } 52 } 53 request.setAttribute("lineChart", "lineChart.jpg"); 54 }MovieDao.java
把数据插入到数据库
1public class MovieDao { 2 3 /** 4 * 把获取的数据，一次性插入 5 * @param listMovie 6 */ 7 public void save(List<Movie> listMovie){ 8 Connection connection = null; 9 PreparedStatement statement = null; 10 11 connection = JdbcUtils.getConnection(); 12 13 try { 14 int i = 1; 15 for(Movie movie : listMovie){ 16 System.out.println("正在插入第"+(i++)+"条数据到数据库ing..."); 17 String sql = " INSERT INTO movie(NAME,TYPES,release_date,score,movieUrl,is_playable) VALUE( ?,?,?,?,?,? ) "; 18 19 statement = connection.prepareStatement(sql); 20 21 statement.setString(1, movie.getName()); 22 statement.setString(2, movie.getTypes()); 23 statement.setString(3, movie.getRelease_date()); 24 statement.setFloat(4, movie.getScore()); 25 statement.setString(5, movie.getMovieUrl()); 26 statement.setString(6, movie.getIs_playable()); 27 28 statement.execute(); 29 } 30 System.out.println("保存数据完成"); 31 } catch (SQLException e) { 32 System.out.println("保存数据出现错误 MovieDao error"); 33 e.printStackTrace(); 34 throw new RuntimeException(e); 35 } finally { 36 try { 37 connection.close(); 38 statement.close(); 39 } catch (SQLException e) { 40 e.printStackTrace(); 41 throw new RuntimeException(e); 42 } 43 } 44 } 45查询所有数据
1 /** 2 * 查询所有数据 3 * @return 4 */ 5 public List<Movie> findAll(){ 6 7 Connection connection = null; 8 PreparedStatement statement = null; 9 ResultSet resultSet = null; 10 11 try { 12 connection = JdbcUtils.getConnection(); 13 14 String sql = " select * from movie "; 15 16 statement = connection.prepareStatement(sql); 17 18 resultSet = statement.executeQuery(); 19 20 List<Movie> list = new ArrayList<Movie>(); 21 while (resultSet.next()) { 22 Movie movie = new Movie(); 23 24 movie.setId(resultSet.getInt("id")); 25 movie.setName(resultSet.getString("name")); 26 movie.setTypes(resultSet.getString("types")); 27 movie.setRelease_date(resultSet.getString("release_date")); 28 movie.setScore(resultSet.getFloat("score")); 29 movie.setMovieUrl(resultSet.getString("movieUrl")); 30 movie.setIs_playable(resultSet.getString("is_playable")); 31 32 list.add(movie); 33 } 34 35 return list; 36 } catch (SQLException e) { 37 e.printStackTrace(); 38 throw new RuntimeException(e); 39 } finally { 40 try { 41 connection.close(); 42 statement.close(); 43 } catch (SQLException e) { 44 e.printStackTrace(); 45 throw new RuntimeException(e); 46 } 47 } 48 }获取不同分数等级的电影数量
1 /** 2 * 统计不同分数级别的电影数量 3 * @return 4 */ 5 public Map<String,Integer> Count(){ 6 7 Connection conn = null; 8 PreparedStatement stmt = null; 9 ResultSet resultSet = null; 10 11 Map<String,Integer> mapCount = new HashMap<String, Integer>(); 12 13 conn = JdbcUtils.getConnection(); 14 String sql = null; 15 String key = null; 16 int i=0; 17 while (i<5) { 18 switch (i) { 19 case 0: 20 sql = "SELECT COUNT(1) FROM movie WHERE score>=9 "; 21 key = "one"; 22 break; 23 case 1: 24 sql = "SELECT COUNT(1) FROM movie WHERE score>=8.5 && score<9 "; 25 key = "two"; 26 break; 27 case 2: 28 sql = "SELECT COUNT(1) FROM movie WHERE score>=8 && score<8.5 "; 29 key = "three"; 30 break; 31 case 3: 32 sql = "SELECT COUNT(1) FROM movie WHERE score>=7.5 && score<8 "; 33 key = "four"; 34 break; 35 case 4: 36 sql = "SELECT COUNT(1) FROM movie WHERE score<7.5 "; 37 key = "five"; 38 break; 39 } 40 try { 41 stmt = conn.prepareStatement(sql); 42 resultSet = stmt.executeQuery(); 43 while (resultSet.next()) { 44 mapCount.put(key, resultSet.getInt(1)); 45 } 46 } catch (SQLException e) { 47 e.printStackTrace(); 48 } 49 i++; 50 } 51 return mapCount; 52 } 53获取每个电影评分的电影数量
1 /** 2 * 统计每个分数对应的数量 3 * @return 4 */ 5 public Map<String,Integer> lineChart(){ 6 7 Connection conn = null; 8 PreparedStatement stmt = null; 9 ResultSet resultSet = null; 10 11 Map<String,Integer> mapCount = new HashMap<String, Integer>(); 12 13 conn = JdbcUtils.getConnection(); 14 String sql = null; 15 int number = 99; 16 for( ; number>=70; number-=1){ 17 //获取9.9 9.1 7.4 ..... 18 String s = number+""; 19 String score = s.charAt(0)+"."+s.charAt(1); 20 sql = "SELECT COUNT(1) FROM movie WHERE score=" + score ; 21 try { 22 stmt = conn.prepareStatement(sql); 23 resultSet = stmt.executeQuery(); 24 while (resultSet.next()) { 25 mapCount.put(score, resultSet.getInt(1)); 26 } 27 } catch (SQLException e) { 28 e.printStackTrace(); 29 } 30 } 31 return mapCount; 32 } 33}
两分钟抓取数据2000+并保存至数据库中，感觉还是挺慢的，有待优化代码
代码源码：
GitHub：https://github.com/YanKuan-IT/DouBanMoviesInfo_DB.git
注：如有什么做的不对的，请指教