View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  
22  package org.opencastproject.workflow.handler.videoeditor;
23  
24  import org.opencastproject.job.api.Job;
25  import org.opencastproject.job.api.JobContext;
26  import org.opencastproject.mediapackage.Catalog;
27  import org.opencastproject.mediapackage.MediaPackage;
28  import org.opencastproject.mediapackage.MediaPackageElement;
29  import org.opencastproject.mediapackage.MediaPackageElementBuilder;
30  import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
31  import org.opencastproject.mediapackage.MediaPackageElementFlavor;
32  import org.opencastproject.mediapackage.Track;
33  import org.opencastproject.mediapackage.selector.TrackSelector;
34  import org.opencastproject.serviceregistry.api.ServiceRegistry;
35  import org.opencastproject.silencedetection.api.SilenceDetectionFailedException;
36  import org.opencastproject.silencedetection.api.SilenceDetectionService;
37  import org.opencastproject.smil.api.SmilException;
38  import org.opencastproject.smil.api.SmilService;
39  import org.opencastproject.smil.entity.api.Smil;
40  import org.opencastproject.smil.entity.media.api.SmilMediaObject;
41  import org.opencastproject.smil.entity.media.container.api.SmilMediaContainer;
42  import org.opencastproject.smil.entity.media.element.api.SmilMediaElement;
43  import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
44  import org.opencastproject.workflow.api.ConfiguredTagsAndFlavors;
45  import org.opencastproject.workflow.api.WorkflowInstance;
46  import org.opencastproject.workflow.api.WorkflowOperationException;
47  import org.opencastproject.workflow.api.WorkflowOperationHandler;
48  import org.opencastproject.workflow.api.WorkflowOperationResult;
49  import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
50  import org.opencastproject.workspace.api.Workspace;
51  
52  import org.apache.commons.io.IOUtils;
53  import org.apache.commons.lang3.BooleanUtils;
54  import org.apache.commons.lang3.StringUtils;
55  import org.osgi.service.component.ComponentContext;
56  import org.osgi.service.component.annotations.Component;
57  import org.osgi.service.component.annotations.Reference;
58  import org.slf4j.Logger;
59  import org.slf4j.LoggerFactory;
60  
61  import java.io.InputStream;
62  import java.net.URI;
63  import java.util.Collection;
64  import java.util.HashMap;
65  import java.util.List;
66  import java.util.Map;
67  import java.util.concurrent.TimeUnit;
68  import java.util.stream.Collectors;
69  
70  /**
71   * workflowoperationhandler for silencedetection executes the silencedetection and adds a SMIL document to the
72   * mediapackage containing the cutting points
73   */
74  @Component(
75      immediate = true,
76      service = WorkflowOperationHandler.class,
77      property = {
78          "service.description=Silence Detection Workflow Operation Handler",
79          "workflow.operation=silence"
80      }
81  )
82  public class SilenceDetectionWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
83  
84    /** Logger */
85    private static final Logger logger = LoggerFactory.getLogger(SilenceDetectionWorkflowOperationHandler.class);
86  
87    /** Name of the configuration option that provides the source flavors we are looking for. */
88    private static final String SOURCE_FLAVORS_PROPERTY = "source-flavors";
89  
90    /** Name of the configuration option that provides the source flavor we are looking for. */
91    private static final String SOURCE_FLAVOR_PROPERTY = "source-flavor";
92  
93    /** Name of the configuration option that provides the smil flavor subtype we will produce. */
94    private static final String SMIL_FLAVOR_SUBTYPE_PROPERTY = "smil-flavor-subtype";
95  
96    /** Name of the configuration option that provides the smil target flavor we will produce. */
97    private static final String SMIL_TARGET_FLAVOR_PROPERTY = "target-flavor";
98  
99    /** Name of the configuration option for track flavors to reference in generated smil. */
100   private static final String REFERENCE_TRACKS_FLAVOR_PROPERTY = "reference-tracks-flavor";
101 
102   /** Name of the configuration option whether to set workflow properties with sum of
103    * segments duration in seconds and relation to the whole track length for each track.*/
104   private static final String EXPORT_SEGMENTS_DURATION = "export-segments-duration";
105 
106   /** Name of the configuration option that provides the smil file name */
107   private static final String TARGET_FILE_NAME = "smil.smil";
108 
109   /** The silence detection service. */
110   private SilenceDetectionService detetionService;
111 
112   /** The smil service for smil parsing. */
113   private SmilService smilService;
114 
115   /** The workspace. */
116   private Workspace workspace;
117 
118   @Override
119   public WorkflowOperationResult start(WorkflowInstance workflowInstance, JobContext context)
120           throws WorkflowOperationException {
121 
122     MediaPackage mp = workflowInstance.getMediaPackage();
123     logger.debug("Start silence detection workflow operation for mediapackage {}", mp.getIdentifier().toString());
124 
125     ConfiguredTagsAndFlavors tagsAndFlavors = getTagsAndFlavors(workflowInstance,
126         Configuration.none, Configuration.many, Configuration.none, Configuration.none);
127     List<MediaPackageElementFlavor> sourceFlavors = tagsAndFlavors.getSrcFlavors();
128     String smilFlavorSubType = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
129             SMIL_FLAVOR_SUBTYPE_PROPERTY));
130     String smilTargetFlavorString = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
131             SMIL_TARGET_FLAVOR_PROPERTY));
132     String exportSegmentsDurationString = StringUtils.trimToNull(
133         workflowInstance.getCurrentOperation().getConfiguration(EXPORT_SEGMENTS_DURATION));
134     boolean exportSegmentsDuration = false;
135 
136     if (StringUtils.isNotBlank(exportSegmentsDurationString)) {
137       try {
138         exportSegmentsDuration = BooleanUtils.toBoolean(exportSegmentsDurationString);
139       } catch (IllegalArgumentException e) {
140         exportSegmentsDuration = false;
141         logger.warn("Unable to parse {} option value {}. Deactivating export of workflow properties.",
142                 EXPORT_SEGMENTS_DURATION, exportSegmentsDurationString);
143       }
144     }
145 
146     MediaPackageElementFlavor smilTargetFlavor = null;
147     if (smilTargetFlavorString != null) {
148       smilTargetFlavor = MediaPackageElementFlavor.parseFlavor(smilTargetFlavorString);
149     }
150 
151     if (sourceFlavors.isEmpty()) {
152       throw new WorkflowOperationException(String.format("No %s or %s have been specified", SOURCE_FLAVOR_PROPERTY,
153               SOURCE_FLAVORS_PROPERTY));
154     }
155     if (smilFlavorSubType == null && smilTargetFlavor == null) {
156       throw new WorkflowOperationException(String.format("No %s or %s have been specified",
157               SMIL_FLAVOR_SUBTYPE_PROPERTY, SMIL_TARGET_FLAVOR_PROPERTY));
158     }
159     if (sourceFlavors != null && smilTargetFlavor != null) {
160       throw new WorkflowOperationException(String.format("Can't use %s and %s together", SOURCE_FLAVORS_PROPERTY,
161               SMIL_TARGET_FLAVOR_PROPERTY));
162     }
163 
164     final String finalSourceFlavors;
165     if (smilTargetFlavor != null) {
166       finalSourceFlavors = sourceFlavors.get(sourceFlavors.size()).toString();
167     } else {
168       finalSourceFlavors = sourceFlavors.stream().map(MediaPackageElementFlavor::toString)
169           .collect(Collectors.joining(","));
170     }
171 
172     String referenceTracksFlavor = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
173             REFERENCE_TRACKS_FLAVOR_PROPERTY));
174     if (referenceTracksFlavor == null) {
175       referenceTracksFlavor = finalSourceFlavors;
176     }
177 
178     TrackSelector trackSelector = new TrackSelector();
179     for (String flavor : asList(finalSourceFlavors)) {
180       trackSelector.addFlavor(flavor);
181     }
182     Collection<Track> sourceTracks = trackSelector.select(mp, false);
183     if (sourceTracks.isEmpty()) {
184       logger.info("No source tracks found, skip silence detection");
185       return createResult(mp, Action.SKIP);
186     }
187 
188     trackSelector = new TrackSelector();
189     for (String flavor : asList(referenceTracksFlavor)) {
190       trackSelector.addFlavor(flavor);
191     }
192     Collection<Track> referenceTracks = trackSelector.select(mp, false);
193     if (referenceTracks.isEmpty()) {
194       // REFERENCE_TRACKS_FLAVOR_PROPERTY was set to wrong value
195       throw new WorkflowOperationException(String.format("No tracks found filtered by flavor(s) '%s'",
196               referenceTracksFlavor));
197     }
198     MediaPackageElementBuilder mpeBuilder = MediaPackageElementBuilderFactory.newInstance().newElementBuilder();
199     Map<String, String> exportWorkflowProperties = new HashMap<>();
200     for (Track sourceTrack : sourceTracks) {
201       // Skip over track with no audio stream
202       if (!sourceTrack.hasAudio()) {
203         logger.info("Skipping silence detection of track {} since it has no audio", sourceTrack);
204         if (exportSegmentsDuration) {
205           exportWorkflowProperties = exportEmptySegmentDuration(exportWorkflowProperties, sourceTrack);
206         }
207         continue;
208       }
209       logger.info("Executing silence detection on track {}", sourceTrack.getIdentifier());
210       try {
211         Job detectionJob = detetionService.detect(sourceTrack,
212                 referenceTracks.toArray(new Track[referenceTracks.size()]));
213         if (!waitForStatus(detectionJob).isSuccess()) {
214           throw new WorkflowOperationException("Silence Detection failed");
215         }
216         Smil smil = smilService.fromXml(detectionJob.getPayload()).getSmil();
217 
218         if (smil.getBody().getMediaElements().isEmpty()) {
219           logger.debug("No segments detected in track {}, skip attaching smil file.", sourceTrack.getIdentifier());
220           if (exportSegmentsDuration) {
221             exportWorkflowProperties = exportEmptySegmentDuration(exportWorkflowProperties, sourceTrack);
222           }
223           continue;
224         }
225 
226         InputStream is = null;
227         try {
228           is = IOUtils.toInputStream(smil.toXML(), "UTF-8");
229           URI smilURI = workspace.put(mp.getIdentifier().toString(), smil.getId(), TARGET_FILE_NAME, is);
230           MediaPackageElementFlavor smilFlavor = smilTargetFlavor;
231           if (smilFlavor == null) {
232             smilFlavor = new MediaPackageElementFlavor(sourceTrack.getFlavor().getType(), smilFlavorSubType);
233           }
234           Catalog catalog = (Catalog) mpeBuilder.elementFromURI(smilURI, MediaPackageElement.Type.Catalog, smilFlavor);
235           catalog.setIdentifier(smil.getId());
236           mp.add(catalog);
237         } catch (Exception ex) {
238           throw new WorkflowOperationException(String.format(
239                   "Failed to put smil into workspace. Silence detection for track %s failed",
240                   sourceTrack.getIdentifier()), ex);
241         } finally {
242           IOUtils.closeQuietly(is);
243         }
244         if (exportSegmentsDuration) {
245           long durationMS = 0;
246           for (SmilMediaObject smilElement : smil.getBody().getMediaElements()) {
247             durationMS += getSegmentDurationMS(smilElement);
248           }
249           String durationWfPropertyName = getDurationWfPropertyName(sourceTrack);
250           exportWorkflowProperties.put(durationWfPropertyName,
251                   Long.toString(TimeUnit.MILLISECONDS.toSeconds(durationMS)));
252           String relationWfPropertyName = getRelationWfPropertyName(sourceTrack);
253           double durationTrackLengthRelation = 0;
254           if (sourceTrack.getDuration() > 0) {
255             durationTrackLengthRelation = (double)durationMS / (double)sourceTrack.getDuration();
256             durationTrackLengthRelation *= 100;
257           }
258           durationTrackLengthRelation = Math.floor(durationTrackLengthRelation);
259           durationTrackLengthRelation = Math.min(100, durationTrackLengthRelation);
260           durationTrackLengthRelation = Math.max(0, durationTrackLengthRelation);
261           exportWorkflowProperties.put(relationWfPropertyName, String.format("%.0f", durationTrackLengthRelation));
262         }
263         logger.info("Finished silence detection on track {}", sourceTrack.getIdentifier());
264       } catch (SilenceDetectionFailedException ex) {
265         throw new WorkflowOperationException(String.format("Failed to create silence detection job for track %s",
266                 sourceTrack.getIdentifier()));
267       } catch (SmilException ex) {
268         throw new WorkflowOperationException(String.format(
269                 "Failed to get smil from silence detection job for track %s", sourceTrack.getIdentifier()));
270       }
271     }
272     logger.debug("Finished silence detection workflow operation for media package {}", mp.getIdentifier());
273     return createResult(mp, exportWorkflowProperties, Action.CONTINUE, 0);
274   }
275 
276   /**
277    * Return first media segment length in milliseconds. If smilElement is a container, look for sub elements and
278    * return duration from the first matching element.
279    * @param smilElement smil media or container element to query duration
280    * @return media duration in milliseconds
281    * @throws SmilException on smil parsing error
282    */
283   protected long getSegmentDurationMS(SmilMediaObject smilElement) throws SmilException {
284     if (smilElement.isContainer()) {
285       for (SmilMediaObject element : ((SmilMediaContainer) smilElement).getElements()) {
286         return getSegmentDurationMS(element);
287       }
288     }
289     SmilMediaElement smilMediaElement = (SmilMediaElement) smilElement;
290     return smilMediaElement.getClipEndMS() - smilMediaElement.getClipBeginMS();
291   }
292 
293   /**
294    * If the track has no audio, still add workflow variables that reflect this
295    * @param properties workflow variable map
296    * @param sourceTrack track without audio
297    * @return The updated workflow variable map
298    */
299   private Map<String, String> exportEmptySegmentDuration(Map<String, String> properties, Track sourceTrack) {
300     String durationWfPropertyName = getDurationWfPropertyName(sourceTrack);
301     properties.put(durationWfPropertyName, Long.toString(0L));
302     String relationWfPropertyName = getRelationWfPropertyName(sourceTrack);
303     properties.put(relationWfPropertyName, String.format("%.0f", 0D));
304     return properties;
305   }
306 
307   private String getDurationWfPropertyName(Track sourceTrack) {
308     return String.format("%s_%s_active_audio_duration",
309         sourceTrack.getFlavor().getType(),
310         sourceTrack.getFlavor().getSubtype());
311   }
312 
313   private String getRelationWfPropertyName(Track sourceTrack) {
314     return String.format("%s_%s_active_audio_duration_percent",
315         sourceTrack.getFlavor().getType(),
316         sourceTrack.getFlavor().getSubtype());
317   }
318 
319   @Override
320   public void activate(ComponentContext cc) {
321     super.activate(cc);
322     logger.info("Registering silence detection workflow operation handler");
323   }
324 
325   @Reference
326   public void setDetectionService(SilenceDetectionService detectionService) {
327     this.detetionService = detectionService;
328   }
329 
330   @Reference
331   public void setSmilService(SmilService smilService) {
332     this.smilService = smilService;
333   }
334 
335   @Reference
336   public void setWorkspace(Workspace workspace) {
337     this.workspace = workspace;
338   }
339 
340   @Reference
341   @Override
342   public void setServiceRegistry(ServiceRegistry serviceRegistry) {
343     super.setServiceRegistry(serviceRegistry);
344   }
345 
346 }