Elasticsearch之pythonAPI简单使用

elasticsearch自动补全建议功能

数据入库操作

ESmapping要求

1PUT music
2{
3    "mappings": {
4        "_doc" : {
5            "properties" : {
6                "suggest" : {
7                    "type" : "completion"
8                },
9                "title" : {
10                    "type": "keyword"
11                }
12            }
13        }
14    }
15}

 DocType类

1from elasticsearch_dsl import DocType, Date, Nested, Boolean, \
2    analyzer, InnerObjectWrapper, Completion, Keyword, Text, Integer
3
4from elasticsearch_dsl.analysis import CustomAnalyzer as _CustomAnalyzer
5
6from elasticsearch_dsl.connections import connections
7connections.create_connection(hosts=["localhost"])
8
9class CustomAnalyzer(_CustomAnalyzer):
10    """
11        避免ik_analyzer参数传递时会报错的问题
12    """
13
14    def get_analysis_definition(self):
15        return {}
16
17
18ik_analyzer = CustomAnalyzer("ik_max_word", filter=["lowercase"])
19
20class ArticleType(DocType):
21
22    suggest = Completion(analyzer=ik_analyzer)
23
24    ... 

 Items类

1from models.es_types import ArticleType
2from elasticsearch_dsl.connections import connections
3es = connections.create_connection(ArticleType._doc_type.using)
4
5
6def gen_suggests(index, info_tuple):
7    # 根据字符串生成搜索建议数组
8    used_words = set()
9    suggests = []
10    for text, weight in info_tuple:
11        if text:
12            # 调用es的analyze接口分析字符串
13            words = es.indices.analyze(index=index, analyzer="ik_max_word", params={'filter':["lowercase"]}, body=text)
14            anylyzed_words = set([r["token"] for r in words["tokens"] if len(r["token"])>1])
15            new_words = anylyzed_words - used_words
16        else:
17            new_words = set()
18
19        if new_words:
20            suggests.append({"input":list(new_words), "weight":weight})
21
22
23class JobBoleArticleItem(scrapy.Item):
24
25    ...
26
27    def save_to_es(self):
28        
29        ...
30
31        article.suggest = gen_suggests(ArticleType._doc_type.index, ((article.title,10),(article.tags, 7)))
32
33        article.save()
34
35        redis_cli.incr("jobbole_count")
36
37        return

 ES搜索语法

1POST myindex/_search?pretty
2{
3    "suggest": {
4        "my-suggest": {
5            "text": "linux",
6            "completion": {
7                "field": "suggest",
8                "fuzzy": {
9                    "fuzziness": 2
10                }
11            }
12        }
13    },
14    "_source": ["title"]  
15}

自动补全建议核心代码

1# django_views中的写法
2
3from search.models import ArticleType
4
5class SearchSuggest(View):
6    def get(self, request):
7        key_words = request.GET.get('s','')
8        re_datas = []
9        if key_words:
10            s = ArticleType.search()
11            s = s.suggest('my_suggest', key_words, completion={
12                "field":"suggest", "fuzzy":{
13                    "fuzziness":2
14                },
15                "size": 10
16            })
17            suggestions = s.execute_suggest()
18            for match in suggestions.my_suggest[0].options:
19                source = match._source
20                re_datas.append(source["title"])
21        return HttpResponse(json.dumps(re_datas), content_type="application/json")

elasticsearch内容搜索功能

数据入库操作

　　和上面一样

搜索核心代码

1# django_views中的写法
2
3from elasticsearch import Elasticsearch
4
5client = Elasticsearch(hosts=["127.0.0.1"])
6
7class SearchView(View):
8    
9    def get(self, request):
10        key_words = request.GET.get("q","")
11        s_type = request.GET.get("s_type", "article")
12        page = request.GET.get("p", "1")
13        try:
14            page = int(page)
15        except:
16            page = 1
17
18        start_time = datetime.now()
19        response = client.search(
20            index= "jobbole",
21            body={
22                "query":{
23                    "multi_match":{
24                        "query":key_words,
25                        "fields":["tags", "title", "content"]
26                    }
27                },
28                "from":(page-1)*10,
29                "size":10,
30                "highlight": {
31                    "pre_tags": ['<span class="keyWord">'],
32                    "post_tags": ['</span>'],
33                    "fields": {
34                        "title": {},
35                        "content": {},
36                    }
37                }
38            }
39        )
40
41        end_time = datetime.now()
42        last_seconds = (end_time-start_time).total_seconds()
43        total_nums = response["hits"]["total"]
44        if (page%10) > 0:
45            page_nums = int(total_nums/10) +1
46        else:
47            page_nums = int(total_nums/10)
48        hit_list = []
49        for hit in response["hits"]["hits"]:
50            hit_dict = {}
51            if "title" in hit["highlight"]:
52                hit_dict["title"] = "".join(hit["highlight"]["title"])
53            else:
54                hit_dict["title"] = hit["_source"]["title"]
55            if "content" in hit["highlight"]:
56                hit_dict["content"] = "".join(hit["highlight"]["content"])[:500]
57            else:
58                hit_dict["content"] = hit["_source"]["content"][:500]
59
60            hit_dict["create_date"] = hit["_source"]["create_date"]
61            hit_dict["url"] = hit["_source"]["url"]
62            hit_dict["score"] = hit["_score"]
63
64            hit_list.append(hit_dict)
65
66        return render(request, "result.html", {"page":page,
67                                            "all_hits":hit_list,
68                                            "key_words":key_words,
69                                            "total_nums":total_nums,
70                                            "page_nums":page_nums,
71                                            "last_seconds":last_seconds
72                                            })

scrapy框架+django框架组合使用

github项目参考

https://github.com/holgerd77/django-dynamic-scraper