View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  
22  package org.opencastproject.elasticsearch.impl;
23  
24  import static org.opencastproject.elasticsearch.impl.IndexSchema.TEXT;
25  
26  import org.opencastproject.elasticsearch.api.SearchQuery;
27  import org.opencastproject.util.DateTimeSupport;
28  
29  import org.apache.commons.lang3.StringUtils;
30  import org.apache.lucene.search.Query;
31  import org.elasticsearch.common.io.stream.StreamOutput;
32  import org.elasticsearch.common.unit.Fuzziness;
33  import org.elasticsearch.common.xcontent.XContentBuilder;
34  import org.elasticsearch.index.query.BoolQueryBuilder;
35  import org.elasticsearch.index.query.MatchAllQueryBuilder;
36  import org.elasticsearch.index.query.MultiMatchQueryBuilder;
37  import org.elasticsearch.index.query.Operator;
38  import org.elasticsearch.index.query.QueryBuilder;
39  import org.elasticsearch.index.query.QueryBuilders;
40  import org.elasticsearch.index.query.QueryRewriteContext;
41  import org.elasticsearch.index.query.QueryShardContext;
42  import org.elasticsearch.index.query.RangeQueryBuilder;
43  import org.elasticsearch.index.query.TermsQueryBuilder;
44  
45  import java.io.IOException;
46  import java.util.ArrayList;
47  import java.util.Arrays;
48  import java.util.Date;
49  import java.util.HashMap;
50  import java.util.HashSet;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.Set;
54  import java.util.stream.Collectors;
55  
56  /**
57   * Opencast implementation of the elastic search query builder.
58   */
59  public abstract class AbstractElasticsearchQueryBuilder<T extends SearchQuery> implements QueryBuilder {
60  
61    /** Term queries on fields */
62    private Map<String, Set<Object>> searchTerms = null;
63  
64    /** Fields that need to match all values */
65    protected List<ValueGroup> groups = null;
66  
67    /** Fields that query a date range */
68    private Set<DateRange> dateRanges = null;
69  
70    /** Filter expression */
71    protected String filter = null;
72  
73    /** Text query */
74    protected String text = null;
75  
76    protected List<String> additionalMultiQueryFields = new ArrayList<>();
77  
78    /** Fuzzy text query */
79    protected boolean fuzzy = false;
80  
81    /** The original search query */
82    private final T query;
83  
84    /** The boolean query */
85    private QueryBuilder queryBuilder = null;
86  
87    /**
88     * Creates a new elastic search query based on the raw query.
89     *
90     * @param query
91     *          the search query
92     */
93    public AbstractElasticsearchQueryBuilder(T query) {
94      this.query = query;
95      buildQuery(query);
96      createQuery();
97    }
98  
99    /**
100    * Returns the original search query.
101    *
102    * @return the search query
103    */
104   public T getQuery() {
105     return query;
106   }
107 
108   public abstract void buildQuery(T query);
109 
110   /**
111    * Create the actual query. We start with a query that matches everything, then move to the boolean conditions,
112    * finally add filter queries.
113    */
114   private void createQuery() {
115 
116     queryBuilder = new MatchAllQueryBuilder();
117 
118     // The boolean query builder
119     BoolQueryBuilder booleanQuery = new BoolQueryBuilder();
120 
121     // Terms
122     if (searchTerms != null) {
123       for (Map.Entry<String, Set<Object>> entry : searchTerms.entrySet()) {
124         booleanQuery.filter(new TermsQueryBuilder(entry.getKey(), entry.getValue().toArray(new Object[0])));
125       }
126       this.queryBuilder = booleanQuery;
127     }
128 
129     // Date ranges
130     if (dateRanges != null) {
131       for (DateRange dr : dateRanges) {
132         booleanQuery.must(dr.getQueryBuilder());
133       }
134       this.queryBuilder = booleanQuery;
135     }
136 
137     // Text
138     if (text != null) {
139       MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(text);
140       queryBuilder.field(TEXT, 1.2f);
141       additionalMultiQueryFields.forEach(field -> queryBuilder.field(field, 1.0f));
142       queryBuilder.type(MultiMatchQueryBuilder.Type.BEST_FIELDS);
143       queryBuilder.operator(Operator.AND);
144       if (fuzzy) {
145         queryBuilder.fuzziness(Fuzziness.AUTO);
146       }
147       booleanQuery.minimumShouldMatch(1);
148       booleanQuery.should(queryBuilder);
149       this.queryBuilder = booleanQuery;
150     }
151 
152     List<QueryBuilder> filters = new ArrayList<>();
153 
154     // Add filtering for AND terms
155     if (groups != null) {
156       for (ValueGroup group : groups) {
157         filters.addAll(group.getFilterBuilders());
158       }
159     }
160 
161     // Filter expressions
162     if (filter != null) {
163       filters.add(QueryBuilders.termQuery(IndexSchema.TEXT, filter));
164     }
165 
166     // Apply the filters
167     if (!filters.isEmpty()) {
168       for (QueryBuilder filter : filters) {
169         booleanQuery.filter(filter);
170       }
171       this.queryBuilder = booleanQuery;
172     }
173 
174   }
175 
176   /**
177    * Stores <code>fieldValue</code> as a search term on the <code>fieldName</code> field.
178    *
179    * @param fieldName
180    *          the field name
181    * @param fieldValues
182    *          the field value
183    */
184   protected void and(String fieldName, Object... fieldValues) {
185 
186     // Make sure the data structures are set up accordingly
187     if (searchTerms == null) {
188       searchTerms = new HashMap<>();
189     }
190 
191     // Fix the field name, just in case
192     fieldName = StringUtils.trim(fieldName);
193 
194     // insert value
195     searchTerms.computeIfAbsent(fieldName, k -> new HashSet<>())
196             .addAll(Arrays.asList(fieldValues));
197   }
198 
199   /**
200    * Stores <code>fieldValue</code> as a search term on the <code>fieldName</code> field.
201    *
202    * @param fieldName
203    *          the field name
204    * @param startDate
205    *          the start date
206    * @param endDate
207    *          the end date
208    */
209   protected void and(String fieldName, Date startDate, Date endDate) {
210 
211     // Fix the field name, just in case
212     fieldName = StringUtils.trim(fieldName);
213 
214     // Make sure the data structures are set up accordingly
215     if (dateRanges == null) {
216       dateRanges = new HashSet<>();
217     }
218 
219     // Add the term
220     dateRanges.add(new DateRange(fieldName, startDate, endDate));
221   }
222 
223   @Override
224   public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
225     return queryBuilder.toXContent(builder, params);
226   }
227 
228   @Override
229   public Query toQuery(QueryShardContext context) throws IOException {
230     return queryBuilder.toQuery(context);
231   }
232 
233   @Override
234   public QueryBuilder queryName(String queryName) {
235     return queryBuilder.queryName(queryName);
236   }
237 
238   @Override
239   public String queryName() {
240     return queryBuilder.queryName();
241   }
242 
243   @Override
244   public float boost() {
245     return queryBuilder.boost();
246   }
247 
248   @Override
249   public QueryBuilder boost(float boost) {
250     return queryBuilder.boost(boost);
251   }
252 
253   @Override
254   public String getName() {
255     return queryBuilder.getName();
256   }
257 
258   @Override
259   public String getWriteableName() {
260     return queryBuilder.getWriteableName();
261   }
262 
263   @Override
264   public void writeTo(StreamOutput out) throws IOException {
265     queryBuilder.writeTo(out);
266   }
267 
268   @Override
269   public QueryBuilder rewrite(QueryRewriteContext queryShardContext) throws IOException {
270     return queryBuilder.rewrite(queryShardContext);
271   }
272 
273   @Override
274   public boolean isFragment() {
275     return queryBuilder.isFragment();
276   }
277 
278   /**
279    * Utility class to hold date range specifications and turn them into elastic search queries.
280    */
281   public static final class DateRange {
282 
283     /** The field name */
284     private String field;
285 
286     /** The start date */
287     private Date startDate;
288 
289     /** The end date */
290     private Date endDate;
291 
292     /**
293      * Creates a new date range specification with the given field name, start and end dates. <code>null</code> may be
294      * passed in for start or end dates that should remain unspecified.
295      *
296      * @param field
297      *          the field name
298      * @param start
299      *          the start date
300      * @param end
301      *          the end date
302      */
303     DateRange(String field, Date start, Date end) {
304       this.field = field;
305       this.startDate = start;
306       this.endDate = end;
307     }
308 
309     /**
310      * Returns the range query that is represented by this date range.
311      *
312      * @return the range query builder
313      */
314     QueryBuilder getQueryBuilder() {
315       RangeQueryBuilder rqb = new RangeQueryBuilder(field);
316       if (startDate != null) {
317         rqb.from(DateTimeSupport.toUTC(startDate.getTime()));
318       }
319       if (endDate != null) {
320         rqb.to(DateTimeSupport.toUTC(endDate.getTime()));
321       }
322       return rqb;
323     }
324 
325     @Override
326     public boolean equals(Object obj) {
327       return obj instanceof DateRange
328               && ((DateRange) obj).field.equals(field);
329     }
330 
331     @Override
332     public int hashCode() {
333       return field.hashCode();
334     }
335 
336   }
337 
338   /**
339    * Stores a group of values which will later be added to the query using AND.
340    */
341   public static final class ValueGroup {
342 
343     /** The field name */
344     private String field;
345 
346     /** The values to store */
347     private Object[] values;
348 
349     /**
350      * Creates a new value group for the given field and values.
351      *
352      * @param field
353      *          the field name
354      * @param values
355      *          the values
356      */
357     public ValueGroup(String field, Object... values) {
358       this.field = field;
359       this.values = values;
360     }
361 
362     /**
363      * Returns the filter that will make sure only documents are returned that match all of the values at once.
364      *
365      * @return the filter builder
366      */
367     List<QueryBuilder> getFilterBuilders() {
368       return Arrays.stream(values)
369               .map((v) -> QueryBuilders.termQuery(field, v.toString()))
370               .collect(Collectors.toList());
371     }
372 
373     @Override
374     public boolean equals(Object obj) {
375       return obj instanceof ValueGroup
376               && ((ValueGroup) obj).field.equals(field);
377     }
378 
379     @Override
380     public int hashCode() {
381       return field.hashCode();
382     }
383 
384   }
385 
386 }