1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.search.endpoint;
23
24 import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
25 import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
26 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
27 import static org.opencastproject.util.RestUtil.R.forbidden;
28 import static org.opencastproject.util.RestUtil.R.noContent;
29 import static org.opencastproject.util.RestUtil.R.notFound;
30 import static org.opencastproject.util.doc.rest.RestParameter.Type.STRING;
31
32 import org.opencastproject.job.api.JobProducer;
33 import org.opencastproject.metadata.dublincore.DublinCore;
34 import org.opencastproject.rest.AbstractJobProducerEndpoint;
35 import org.opencastproject.search.api.SearchException;
36 import org.opencastproject.search.api.SearchResult;
37 import org.opencastproject.search.api.SearchResultList;
38 import org.opencastproject.search.api.SearchService;
39 import org.opencastproject.search.impl.SearchServiceImpl;
40 import org.opencastproject.search.impl.SearchServiceIndex;
41 import org.opencastproject.security.api.Role;
42 import org.opencastproject.security.api.SecurityConstants;
43 import org.opencastproject.security.api.SecurityService;
44 import org.opencastproject.security.api.UnauthorizedException;
45 import org.opencastproject.security.urlsigning.exception.UrlSigningException;
46 import org.opencastproject.security.urlsigning.service.UrlSigningService;
47 import org.opencastproject.security.urlsigning.utils.UrlSigningServiceOsgiUtil;
48 import org.opencastproject.serviceregistry.api.ServiceRegistry;
49 import org.opencastproject.util.NotFoundException;
50 import org.opencastproject.util.doc.rest.RestParameter;
51 import org.opencastproject.util.doc.rest.RestQuery;
52 import org.opencastproject.util.doc.rest.RestResponse;
53 import org.opencastproject.util.doc.rest.RestService;
54
55 import com.google.gson.Gson;
56
57 import org.apache.commons.lang3.StringUtils;
58 import org.apache.commons.lang3.math.NumberUtils;
59 import org.elasticsearch.index.query.Operator;
60 import org.elasticsearch.index.query.QueryBuilders;
61 import org.elasticsearch.search.SearchHit;
62 import org.elasticsearch.search.builder.SearchSourceBuilder;
63 import org.elasticsearch.search.sort.SortOrder;
64 import org.osgi.service.component.annotations.Component;
65 import org.osgi.service.component.annotations.Reference;
66 import org.osgi.service.jaxrs.whiteboard.propertytypes.JaxrsResource;
67 import org.slf4j.Logger;
68 import org.slf4j.LoggerFactory;
69
70 import java.util.Arrays;
71 import java.util.Collections;
72 import java.util.List;
73 import java.util.Map;
74 import java.util.stream.Collectors;
75
76 import javax.servlet.http.HttpServletResponse;
77 import javax.ws.rs.FormParam;
78 import javax.ws.rs.GET;
79 import javax.ws.rs.POST;
80 import javax.ws.rs.Path;
81 import javax.ws.rs.Produces;
82 import javax.ws.rs.QueryParam;
83 import javax.ws.rs.WebApplicationException;
84 import javax.ws.rs.core.MediaType;
85 import javax.ws.rs.core.Response;
86
87
88
89
90 @Path("/search")
91 @RestService(
92 name = "search",
93 title = "Search Service",
94 abstractText = "This service indexes and queries available (distributed) episodes.",
95 notes = {
96 "All paths above are relative to the REST endpoint base (something like http://your.server/files)",
97 "If the service is down or not working it will return a status 503, this means the the "
98 + "underlying service is not working and is either restarting or has failed",
99 "A status code 500 means a general failure has occurred which is not recoverable and was "
100 + "not anticipated. In other words, there is a bug! You should file an error report "
101 + "with your server logs from the time when the error occurred: "
102 + "<a href=\"https://github.com/opencast/opencast/issues\">Opencast Issue Tracker</a>"
103 }
104 )
105 @Component(
106 immediate = true,
107 service = SearchRestService.class,
108 property = {
109 "service.description=Search REST Endpoint",
110 "opencast.service.type=org.opencastproject.search",
111 "opencast.service.path=/search",
112 "opencast.service.jobproducer=true"
113 }
114 )
115 @JaxrsResource
116 public class SearchRestService extends AbstractJobProducerEndpoint {
117
118 private static final Logger logger = LoggerFactory.getLogger(SearchRestService.class);
119
120
121 protected SearchServiceImpl searchService;
122
123
124 protected SearchServiceIndex searchIndex;
125
126
127 private ServiceRegistry serviceRegistry;
128
129 private SecurityService securityService;
130
131 private final Gson gson = new Gson();
132
133 private UrlSigningService urlSigningService;
134
135 @GET
136 @Path("series.json")
137 @Produces(MediaType.APPLICATION_JSON)
138 @RestQuery(
139 name = "get_series",
140 description = "Search for series matching the query parameters.",
141 restParameters = {
142 @RestParameter(
143 name = "id",
144 isRequired = false,
145 type = RestParameter.Type.STRING,
146 description = "The series ID. If the additional boolean parameter \"episodes\" is \"true\", "
147 + "the result set will include this series episodes."
148 ),
149 @RestParameter(
150 name = "q",
151 isRequired = false,
152 type = RestParameter.Type.STRING,
153 description = "Any series that matches this free-text query."
154 ),
155 @RestParameter(
156 name = "sort",
157 isRequired = false,
158 type = RestParameter.Type.STRING,
159 description = "The sort order. May include any of the following dublin core metadata: "
160 + "identifier, title, contributor, creator, created, modified. "
161 + "Add ' asc' or ' desc' to specify the sort order (e.g. 'title desc')."
162 ),
163 @RestParameter(
164 name = "limit",
165 isRequired = false,
166 type = RestParameter.Type.INTEGER,
167 defaultValue = "20",
168 description = "The maximum number of items to return per page."
169 ),
170 @RestParameter(
171 name = "offset",
172 isRequired = false,
173 type = RestParameter.Type.INTEGER,
174 defaultValue = "0",
175 description = "The page number."
176 )
177 },
178 responses = {
179 @RestResponse(
180 description = "The request was processed successfully.",
181 responseCode = HttpServletResponse.SC_OK
182 )
183 },
184 returnDescription = "The search results, formatted as XML or JSON."
185 )
186 public Response getSeries(
187 @QueryParam("id") String id,
188 @QueryParam("q") String text,
189 @QueryParam("sort") String sort,
190 @QueryParam("limit") String limit,
191 @QueryParam("offset") String offset
192 ) throws SearchException {
193
194 final var org = securityService.getOrganization().getId();
195 final var type = SearchService.IndexEntryType.Series.name();
196 final var query = QueryBuilders.boolQuery()
197 .must(QueryBuilders.termQuery(SearchResult.ORG, org))
198 .must(QueryBuilders.termQuery(SearchResult.TYPE, type))
199 .mustNot(QueryBuilders.existsQuery(SearchResult.DELETED_DATE));
200
201 if (StringUtils.isNotEmpty(id)) {
202 query.must(QueryBuilders.idsQuery().addIds(id));
203 }
204
205 if (StringUtils.isNotEmpty(text)) {
206 query.must(QueryBuilders.wildcardQuery("fulltext", "*" + text.toLowerCase() + "*"));
207 }
208
209 var user = securityService.getUser();
210 var orgAdminRole = securityService.getOrganization().getAdminRole();
211 if (!user.hasRole(SecurityConstants.GLOBAL_ADMIN_ROLE) && !user.hasRole(orgAdminRole)) {
212 query.must(QueryBuilders.termsQuery(
213 SearchResult.INDEX_ACL + ".read",
214 user.getRoles().stream().map(Role::getName).collect(Collectors.toList())
215 ));
216 }
217
218 var size = NumberUtils.toInt(limit, 20);
219 var from = NumberUtils.toInt(offset);
220 if (size < 0 || from < 0) {
221 return Response.status(Response.Status.BAD_REQUEST)
222 .entity("Limit and offset may not be negative.")
223 .build();
224 }
225 var searchSource = new SearchSourceBuilder()
226 .query(query)
227 .from(from)
228 .size(size);
229
230 if (StringUtils.isNotEmpty(sort)) {
231 var sortParam = StringUtils.split(sort.toLowerCase());
232 var validSort = Arrays.asList("identifier", "title", "contributor", "creator", "created", "modified")
233 .contains(sortParam[0]);
234 var validOrder = sortParam.length < 2 || Arrays.asList("asc", "desc").contains(sortParam[1]);
235 if (sortParam.length > 2 || !validSort || !validOrder) {
236 return Response.status(Response.Status.BAD_REQUEST)
237 .entity("Invalid sort parameter")
238 .build();
239 }
240 var order = SortOrder.fromString(sortParam.length > 1 ? sortParam[1] : "asc");
241 if ("modified".equals(sortParam[0])) {
242 searchSource.sort(sortParam[0], order);
243 } else {
244 searchSource.sort(SearchResult.DUBLINCORE + "." + sortParam[0], order);
245 }
246 }
247
248 var hits = searchIndex.search(searchSource).getHits();
249 var result = Arrays.stream(hits.getHits())
250 .map(SearchHit::getSourceAsMap)
251 .peek(hit -> hit.remove(SearchResult.TYPE))
252 .collect(Collectors.toList());
253
254 var total = hits.getTotalHits().value;
255 var json = gson.toJsonTree(Map.of(
256 "offset", from,
257 "total", total,
258 "result", result,
259 "limit", size));
260 return Response.ok(gson.toJson(json)).build();
261
262 }
263
264 @GET
265 @Path("episode.json")
266 @Produces(MediaType.APPLICATION_JSON)
267 @RestQuery(
268 name = "search_episodes",
269 description = "Search for episodes matching the query parameters.",
270 restParameters = {
271 @RestParameter(
272 name = "id",
273 isRequired = false,
274 type = RestParameter.Type.STRING,
275 description = "The ID of the single episode to be returned, if it exists."
276 ),
277 @RestParameter(
278 name = "q",
279 isRequired = false,
280 type = RestParameter.Type.STRING,
281 description = "Any episode that matches this free-text query."
282 ),
283 @RestParameter(
284 name = "sid",
285 isRequired = false,
286 type = RestParameter.Type.STRING,
287 description = "Any episode that belongs to specified series id."
288 ),
289 @RestParameter(
290 name = "sname",
291 isRequired = false,
292 type = RestParameter.Type.STRING,
293 description = "Any episode that belongs to specified series name (note that the "
294 + "specified series name must be unique)."
295 ),
296 @RestParameter(
297 name = "sort",
298 isRequired = false,
299 type = RestParameter.Type.STRING,
300 description = "The sort order. May include any of the following dublin core metadata: "
301 + "title, contributor, creator, created, modified. "
302 + "Add ' asc' or ' desc' to specify the sort order (e.g. 'title desc')."
303 ),
304 @RestParameter(
305 name = "limit",
306 isRequired = false,
307 type = RestParameter.Type.INTEGER,
308 defaultValue = "20",
309 description = "The maximum number of items to return per page. Limited to 250 for non-admins."
310 ),
311 @RestParameter(
312 name = "offset",
313 isRequired = false,
314 type = RestParameter.Type.INTEGER,
315 defaultValue = "0",
316 description = "The page number."
317 ),
318 @RestParameter(
319 name = "sign",
320 isRequired = false,
321 type = RestParameter.Type.BOOLEAN,
322 defaultValue = "true",
323 description = "If results are to be signed"
324 )
325 },
326 responses = {
327 @RestResponse(
328 description = "The request was processed successfully.",
329 responseCode = HttpServletResponse.SC_OK
330 )
331 },
332 returnDescription = "The search results, formatted as xml or json."
333 )
334 public Response getEpisodes(
335 @QueryParam("id") String id,
336 @QueryParam("q") String text,
337 @QueryParam("sid") String seriesId,
338 @QueryParam("sname") String seriesName,
339 @QueryParam("sort") String sort,
340 @QueryParam("limit") String limit,
341 @QueryParam("offset") String offset,
342 @QueryParam("sign") String sign
343 ) throws SearchException {
344
345
346 if (StringUtils.isNoneEmpty(seriesName, seriesId)) {
347 return Response.status(Response.Status.BAD_REQUEST)
348 .entity("invalid request, both 'sid' and 'sname' specified")
349 .build();
350 }
351 final var org = securityService.getOrganization().getId();
352 final var type = SearchService.IndexEntryType.Episode.name();
353
354 boolean snameNotFound = false;
355 List<String> series = Collections.emptyList();
356 if (StringUtils.isNotEmpty(seriesName)) {
357 var seriesSearchSource = new SearchSourceBuilder().query(QueryBuilders.boolQuery()
358 .filter(QueryBuilders.termQuery(SearchResult.ORG, org))
359 .filter(QueryBuilders.termQuery(SearchResult.TYPE, SearchService.IndexEntryType.Series))
360 .filter(QueryBuilders.termQuery(SearchResult.DUBLINCORE + ".title", seriesName))
361 .mustNot(QueryBuilders.existsQuery(SearchResult.DELETED_DATE)));
362 series = searchService.search(seriesSearchSource).getHits().stream()
363 .map(h -> h.getDublinCore().getFirst(DublinCore.PROPERTY_IDENTIFIER))
364 .collect(Collectors.toList());
365
366 if (series.isEmpty()) {
367 snameNotFound = true;
368 }
369 }
370
371 var query = QueryBuilders.boolQuery()
372 .filter(QueryBuilders.termQuery(SearchResult.ORG, org))
373 .filter(QueryBuilders.termQuery(SearchResult.TYPE, type))
374 .mustNot(QueryBuilders.existsQuery(SearchResult.DELETED_DATE));
375
376 if (StringUtils.isNotEmpty(id)) {
377 query.filter(QueryBuilders.idsQuery().addIds(id));
378 }
379
380 if (StringUtils.isNotEmpty(seriesId)) {
381 series = Collections.singletonList(seriesId);
382 }
383 if (!series.isEmpty()) {
384 if (series.size() == 1) {
385 query.must(QueryBuilders.termQuery(SearchResult.DUBLINCORE + ".isPartOf", series.get(0)));
386 } else {
387 var seriesQuery = QueryBuilders.boolQuery();
388 for (var sid : series) {
389 seriesQuery.should(QueryBuilders.termQuery(SearchResult.DUBLINCORE + ".isPartOf", sid));
390 }
391 query.must(seriesQuery);
392 }
393 }
394
395 if (StringUtils.isNotEmpty(text)) {
396 query.minimumShouldMatch(1);
397 query.should(
398 QueryBuilders.matchQuery("fulltext", text)
399 .fuzziness("AUTO")
400 .operator(Operator.AND)
401 );
402 }
403
404 var user = securityService.getUser();
405 var orgAdminRole = securityService.getOrganization().getAdminRole();
406 var admin = user.hasRole(SecurityConstants.GLOBAL_ADMIN_ROLE) || user.hasRole(orgAdminRole);
407 if (!admin) {
408 query.must(QueryBuilders.termsQuery(
409 SearchResult.INDEX_ACL + ".read",
410 user.getRoles().stream().map(Role::getName).collect(Collectors.toList())
411 ));
412 }
413
414 logger.debug("limit: {}, offset: {}", limit, offset);
415
416 var size = NumberUtils.toInt(limit, 20);
417 var from = NumberUtils.toInt(offset);
418 if (size < 0 || from < 0) {
419 return Response.status(Response.Status.BAD_REQUEST)
420 .entity("Limit and offset may not be negative.")
421 .build();
422 }
423 if (!admin && size > 250) {
424 return Response.status(Response.Status.BAD_REQUEST)
425 .entity("Only admins are allowed to request more than 250 items.")
426 .build();
427 }
428
429 var searchSource = new SearchSourceBuilder()
430 .query(query)
431 .from(from)
432 .size(size);
433
434 if (StringUtils.isNotEmpty(sort)) {
435 var sortParam = StringUtils.split(sort.toLowerCase());
436 var validSort = Arrays.asList("title", "contributor", "creator", "created", "modified").contains(sortParam[0]);
437 var validOrder = sortParam.length < 2 || Arrays.asList("asc", "desc").contains(sortParam[1]);
438 if (sortParam.length > 2 || !validSort || !validOrder) {
439 return Response.status(Response.Status.BAD_REQUEST)
440 .entity("Invalid sort parameter")
441 .build();
442 }
443 var order = SortOrder.fromString(sortParam.length > 1 ? sortParam[1] : "asc");
444 if ("modified".equals(sortParam[0])) {
445 searchSource.sort(sortParam[0], order);
446 } else {
447 searchSource.sort(SearchResult.DUBLINCORE + "." + sortParam[0], order);
448 }
449 }
450
451 List<Map<String, Object>> result = null;
452 long total = 0;
453 if (snameNotFound) {
454 result = Collections.emptyList();
455 } else {
456 SearchResultList hits = searchService.search(searchSource);
457 result = hits.getHits().stream()
458 .map(SearchResult::dehydrateForREST)
459 .collect(Collectors.toList());
460
461
462 if (!"false".equals(sign) && this.urlSigningService != null) {
463 this.findURLsAndSign(result);
464 }
465
466 total = hits.getTotalHits();
467 }
468 var json = gson.toJson(Map.of(
469 "offset", from,
470 "total", total,
471 "result", result,
472 "limit", size));
473
474 return Response.ok(json).build();
475 }
476
477 @POST
478 @Path("updateIndex")
479 @RestQuery(name = "updateIndex",
480 description = "Trigger search index update for event. The usage of this is limited to global administrators.",
481 restParameters = {
482 @RestParameter(
483 name = "id",
484 isRequired = true,
485 type = STRING,
486 description = "The event ID to trigger an index update for.")},
487 responses = {
488 @RestResponse(
489 description = "Update successfully triggered.",
490 responseCode = SC_NO_CONTENT),
491 @RestResponse(
492 description = "Not allowed to trigger update.",
493 responseCode = SC_FORBIDDEN),
494 @RestResponse(
495 description = "No such event found.",
496 responseCode = SC_NOT_FOUND)},
497 returnDescription = "No content is returned.")
498 public Response indexUpdate(@FormParam("id") final String id) {
499 try {
500 searchIndex.indexMediaPackage(id);
501 return noContent();
502 } catch (UnauthorizedException e) {
503 return forbidden();
504 } catch (NotFoundException e) {
505 return notFound();
506 } catch (Exception e) {
507 throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
508 }
509 }
510
511
512
513
514
515 private void findURLsAndSign(Object obj) {
516 if (obj instanceof Map) {
517 Map<String, Object> map = (Map<String, Object>) obj;
518 for (Map.Entry<String, Object> entry : map.entrySet()) {
519 if (entry.getKey().equals("url") && entry.getValue() instanceof String) {
520 String urlToSign = (String) entry.getValue();
521 if (this.urlSigningService.accepts(urlToSign)) {
522 try {
523 String signedUrl = this.urlSigningService.sign(
524 urlToSign,
525 UrlSigningServiceOsgiUtil.DEFAULT_URL_SIGNING_EXPIRE_DURATION,
526 null,
527 null);
528 map.put(entry.getKey(), signedUrl);
529 } catch (UrlSigningException e) {
530 logger.debug("Unable to sign url '{}'.", urlToSign);
531 }
532 }
533 } else {
534 findURLsAndSign(entry.getValue());
535 }
536 }
537 } else if (obj instanceof List) {
538 for (Object item : (List<?>) obj) {
539 findURLsAndSign(item);
540 }
541 }
542 }
543
544
545
546
547 @Override
548 public JobProducer getService() {
549 return searchService;
550 }
551
552
553
554
555
556
557
558 @Reference
559 public void setSearchService(SearchServiceImpl searchService) {
560 this.searchService = searchService;
561 }
562
563 @Reference
564 public void setSearchIndex(SearchServiceIndex searchIndex) {
565 this.searchIndex = searchIndex;
566 }
567
568 @Reference
569 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
570 this.serviceRegistry = serviceRegistry;
571 }
572
573 @Override
574 public ServiceRegistry getServiceRegistry() {
575 return serviceRegistry;
576 }
577
578 @Reference
579 public void setSecurityService(SecurityService securityService) {
580 this.securityService = securityService;
581 }
582
583 @Reference
584 void setUrlSigningService(UrlSigningService service) {
585 this.urlSigningService = service;
586 }
587
588 }