热更新
在上一节《 IK分词器配置文件讲解以及自定义词库》自定义词库，每次都是在es的扩展词典中，手动添加新词语，很坑
（1）每次添加完，都要重启es才能生效，非常麻烦
（2）es是分布式的，可能有数百个节点，你不能每次都一个一个节点上面去修改
es不停机，直接我们在外部某个地方添加新的词语，es中立即热加载到这些新词语
热更新的方案
（1）修改ik分词器源码，然后手动支持从mysql中每隔一定时间，自动加载新的词库
（2）基于ik分词器原生支持的热更新方案，部署一个web服务器，提供一个http接口，通过modified和tag两个http响应头，来提供词语的热更新
用第一种方案，第二种，ik git社区官方都不建议采用，觉得不太稳定
1、下载源码
https://github.com/medcl/elasticsearch-analysis-ik/tree/v5.2.0
ik分词器，是个标准的java maven工程，直接导入eclipse就可以看到源码
2、修改源码
Dictionary单例类的初始化方法initial，在这里需要创建一个我们自定义的线程，并且启动它
1/** 2 * 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化 3 * 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段 4 * 5 * @return Dictionary 6 */ 7public static synchronized Dictionary initial(Configuration cfg) { 8 if (singleton == null) { 9 synchronized (Dictionary.class) { 10 if (singleton == null) { 11 12 13 singleton = new Dictionary(cfg); 14 singleton.loadMainDict(); 15 singleton.loadSurnameDict(); 16 singleton.loadQuantifierDict(); 17 singleton.loadSuffixDict(); 18 singleton.loadPrepDict(); 19 singleton.loadStopWordDict(); 20 21 new Thread(new HotDictReloadThread()).start(); 22 23 if(cfg.isEnableRemoteDict()){ 24 // 建立监控线程 25 for (String location : singleton.getRemoteExtDictionarys()) { 26 // 10 秒是初始延迟可以修改的 60是间隔时间 单位秒 27 pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS); 28 } 29 for (String location : singleton.getRemoteExtStopWordDictionarys()) { 30 pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS); 31 } 32 } 33 34 35 return singleton; 36 } 37 } 38 } 39 return singleton; 40}
HotDictReloadThread类：就是死循环，不断调用Dictionary.getSingleton().reLoadMainDict()，去重新加载词典
1public class HotDictReloadThread implements Runnable { 2 3private static final Logger logger = ESLoggerFactory.getLogger(HotDictReloadThread.class.getName()); 4 5@Override 6public void run() { 7 while(true) { 8 logger.info("[==========]reload hot dict from mysql......"); 9 Dictionary.getSingleton().reLoadMainDict(); 10 } 11} 12 13}
Dictionary类：更新词典 this.loadMySQLExtDict()
1/** 2 * 加载主词典及扩展词典 3 */ 4private void loadMainDict() { 5 // 建立一个主词典实例 6 _MainDict = new DictSegment((char) 0); 7 8 // 读取主词典文件 9 Path file = PathUtils.get(getDictRoot(), Dictionary.PATH_DIC_MAIN); 10 11 InputStream is = null; 12 try { 13 is = new FileInputStream(file.toFile()); 14 } catch (FileNotFoundException e) { 15 logger.error(e.getMessage(), e); 16 } 17 18 try { 19 BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512); 20 String theWord = null; 21 do { 22 theWord = br.readLine(); 23 if (theWord != null && !"".equals(theWord.trim())) { 24 _MainDict.fillSegment(theWord.trim().toCharArray()); 25 } 26 } while (theWord != null); 27 28 } catch (IOException e) { 29 logger.error("ik-analyzer", e); 30 31 } finally { 32 try { 33 if (is != null) { 34 is.close(); 35 is = null; 36 } 37 } catch (IOException e) { 38 logger.error("ik-analyzer", e); 39 } 40 } 41 // 加载扩展词典 42 this.loadExtDict(); 43 // 加载远程自定义词库 44 this.loadRemoteExtDict(); 45 // 从mysql加载词典 46 this.loadMySQLExtDict(); 47} 48 49/** 50 * 从mysql加载热更新词典 51 */ 52private void loadMySQLExtDict() { 53 Connection conn = null; 54 Statement stmt = null; 55 ResultSet rs = null; 56 57 try { 58 Path file = PathUtils.get(getDictRoot(), "jdbc-reload.properties"); 59 prop.load(new FileInputStream(file.toFile())); 60 61 logger.info("[==========]jdbc-reload.properties"); 62 for(Object key : prop.keySet()) { 63 logger.info("[==========]" + key + "=" + prop.getProperty(String.valueOf(key))); 64 } 65 66 logger.info("[==========]query hot dict from mysql, " + prop.getProperty("jdbc.reload.sql") + "......"); 67 68 conn = DriverManager.getConnection( 69 prop.getProperty("jdbc.url"), 70 prop.getProperty("jdbc.user"), 71 prop.getProperty("jdbc.password")); 72 stmt = conn.createStatement(); 73 rs = stmt.executeQuery(prop.getProperty("jdbc.reload.sql")); 74 75 while(rs.next()) { 76 String theWord = rs.getString("word"); 77 logger.info("[==========]hot word from mysql: " + theWord); 78 _MainDict.fillSegment(theWord.trim().toCharArray()); 79 } 80 81 Thread.sleep(Integer.valueOf(String.valueOf(prop.get("jdbc.reload.interval")))); 82 } catch (Exception e) { 83 logger.error("erorr", e); 84 } finally { 85 if(rs != null) { 86 try { 87 rs.close(); 88 } catch (SQLException e) { 89 logger.error("error", e); 90 } 91 } 92 if(stmt != null) { 93 try { 94 stmt.close(); 95 } catch (SQLException e) { 96 logger.error("error", e); 97 } 98 } 99 if(conn != null) { 100 try { 101 conn.close(); 102 } catch (SQLException e) { 103 logger.error("error", e); 104 } 105 } 106 } 107}
Dictionary类：更新分词 this.loadMySQLStopwordDict();
1/** 2 * 从mysql加载停用词 3 */ 4private void loadMySQLStopwordDict() { 5 Connection conn = null; 6 Statement stmt = null; 7 ResultSet rs = null; 8 9 try { 10 Path file = PathUtils.get(getDictRoot(), "jdbc-reload.properties"); 11 prop.load(new FileInputStream(file.toFile())); 12 13 logger.info("[==========]jdbc-reload.properties"); 14 for(Object key : prop.keySet()) { 15 logger.info("[==========]" + key + "=" + prop.getProperty(String.valueOf(key))); 16 } 17 18 logger.info("[==========]query hot stopword dict from mysql, " + prop.getProperty("jdbc.reload.stopword.sql") + "......"); 19 20 conn = DriverManager.getConnection( 21 prop.getProperty("jdbc.url"), 22 prop.getProperty("jdbc.user"), 23 prop.getProperty("jdbc.password")); 24 stmt = conn.createStatement(); 25 rs = stmt.executeQuery(prop.getProperty("jdbc.reload.stopword.sql")); 26 27 while(rs.next()) { 28 String theWord = rs.getString("word"); 29 logger.info("[==========]hot stopword from mysql: " + theWord); 30 _StopWords.fillSegment(theWord.trim().toCharArray()); 31 } 32 33 Thread.sleep(Integer.valueOf(String.valueOf(prop.get("jdbc.reload.interval")))); 34 } catch (Exception e) { 35 logger.error("erorr", e); 36 } finally { 37 if(rs != null) { 38 try { 39 rs.close(); 40 } catch (SQLException e) { 41 logger.error("error", e); 42 } 43 } 44 if(stmt != null) { 45 try { 46 stmt.close(); 47 } catch (SQLException e) { 48 logger.error("error", e); 49 } 50 } 51 if(conn != null) { 52 try { 53 conn.close(); 54 } catch (SQLException e) { 55 logger.error("error", e); 56 } 57 } 58 } 59}
配置
1jdbc.url=jdbc:mysql://localhost:3306/test?serverTimezone=GMT 2jdbc.user=root 3jdbc.password=root 4jdbc.reload.sql=select word from hot_words 5jdbc.reload.stopword.sql=select stopword as word from hot_stopwords 6jdbc.reload.interval=1000
3、mvn package打包代码
target\releases\elasticsearch-analysis-ik-5.2.0.zip
4、解压缩ik压缩包
将mysql驱动jar，放入ik的目录下
5、重启es
6、在mysql中添加词库与停用词
7、kibana分词验证
1GET /my_index/_analyze 2{ 3 "text": "一人饮酒醉", 4 "analyzer": "ik_max_word" 5} 6 7{ 8 "tokens": [ 9 { 10 "token": "一人饮酒醉", 11 "start_offset": 0, 12 "end_offset": 5, 13 "type": "CN_WORD", 14 "position": 0 15 }, 16 { 17 "token": "一人", 18 "start_offset": 0, 19 "end_offset": 2, 20 "type": "CN_WORD", 21 "position": 1 22 }, 23 { 24 "token": "一", 25 "start_offset": 0, 26 "end_offset": 1, 27 "type": "TYPE_CNUM", 28 "position": 2 29 }, 30 { 31 "token": "人", 32 "start_offset": 1, 33 "end_offset": 2, 34 "type": "COUNT", 35 "position": 3 36 }, 37 { 38 "token": "饮酒", 39 "start_offset": 2, 40 "end_offset": 4, 41 "type": "CN_WORD", 42 "position": 4 43 }, 44 { 45 "token": "饮", 46 "start_offset": 2, 47 "end_offset": 3, 48 "type": "CN_WORD", 49 "position": 5 50 }, 51 { 52 "token": "酒醉", 53 "start_offset": 3, 54 "end_offset": 5, 55 "type": "CN_WORD", 56 "position": 6 57 }, 58 { 59 "token": "酒", 60 "start_offset": 3, 61 "end_offset": 4, 62 "type": "CN_WORD", 63 "position": 7 64 }, 65 { 66 "token": "醉", 67 "start_offset": 4, 68 "end_offset": 5, 69 "type": "CN_WORD", 70 "position": 8 71 } 72 ] 73}