1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.workflow.handler.videoeditor;
23
24 import org.opencastproject.job.api.Job;
25 import org.opencastproject.job.api.JobContext;
26 import org.opencastproject.mediapackage.Catalog;
27 import org.opencastproject.mediapackage.MediaPackage;
28 import org.opencastproject.mediapackage.MediaPackageElement;
29 import org.opencastproject.mediapackage.MediaPackageElementBuilder;
30 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
31 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
32 import org.opencastproject.mediapackage.Track;
33 import org.opencastproject.mediapackage.selector.TrackSelector;
34 import org.opencastproject.serviceregistry.api.ServiceRegistry;
35 import org.opencastproject.silencedetection.api.SilenceDetectionFailedException;
36 import org.opencastproject.silencedetection.api.SilenceDetectionService;
37 import org.opencastproject.smil.api.SmilException;
38 import org.opencastproject.smil.api.SmilService;
39 import org.opencastproject.smil.entity.api.Smil;
40 import org.opencastproject.smil.entity.media.api.SmilMediaObject;
41 import org.opencastproject.smil.entity.media.container.api.SmilMediaContainer;
42 import org.opencastproject.smil.entity.media.element.api.SmilMediaElement;
43 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
44 import org.opencastproject.workflow.api.ConfiguredTagsAndFlavors;
45 import org.opencastproject.workflow.api.WorkflowInstance;
46 import org.opencastproject.workflow.api.WorkflowOperationException;
47 import org.opencastproject.workflow.api.WorkflowOperationHandler;
48 import org.opencastproject.workflow.api.WorkflowOperationResult;
49 import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
50 import org.opencastproject.workspace.api.Workspace;
51
52 import org.apache.commons.io.IOUtils;
53 import org.apache.commons.lang3.BooleanUtils;
54 import org.apache.commons.lang3.StringUtils;
55 import org.osgi.service.component.ComponentContext;
56 import org.osgi.service.component.annotations.Component;
57 import org.osgi.service.component.annotations.Reference;
58 import org.slf4j.Logger;
59 import org.slf4j.LoggerFactory;
60
61 import java.io.InputStream;
62 import java.net.URI;
63 import java.util.Collection;
64 import java.util.HashMap;
65 import java.util.List;
66 import java.util.Map;
67 import java.util.concurrent.TimeUnit;
68 import java.util.stream.Collectors;
69
70
71
72
73
74 @Component(
75 immediate = true,
76 service = WorkflowOperationHandler.class,
77 property = {
78 "service.description=Silence Detection Workflow Operation Handler",
79 "workflow.operation=silence"
80 }
81 )
82 public class SilenceDetectionWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
83
84
85 private static final Logger logger = LoggerFactory.getLogger(SilenceDetectionWorkflowOperationHandler.class);
86
87
88 private static final String SOURCE_FLAVORS_PROPERTY = "source-flavors";
89
90
91 private static final String SOURCE_FLAVOR_PROPERTY = "source-flavor";
92
93
94 private static final String SMIL_FLAVOR_SUBTYPE_PROPERTY = "smil-flavor-subtype";
95
96
97 private static final String SMIL_TARGET_FLAVOR_PROPERTY = "target-flavor";
98
99
100 private static final String REFERENCE_TRACKS_FLAVOR_PROPERTY = "reference-tracks-flavor";
101
102
103
104 private static final String EXPORT_SEGMENTS_DURATION = "export-segments-duration";
105
106
107 private static final String TARGET_FILE_NAME = "smil.smil";
108
109
110 private SilenceDetectionService detetionService;
111
112
113 private SmilService smilService;
114
115
116 private Workspace workspace;
117
118 @Override
119 public WorkflowOperationResult start(WorkflowInstance workflowInstance, JobContext context)
120 throws WorkflowOperationException {
121
122 MediaPackage mp = workflowInstance.getMediaPackage();
123 logger.debug("Start silence detection workflow operation for mediapackage {}", mp.getIdentifier().toString());
124
125 ConfiguredTagsAndFlavors tagsAndFlavors = getTagsAndFlavors(workflowInstance,
126 Configuration.none, Configuration.many, Configuration.none, Configuration.none);
127 List<MediaPackageElementFlavor> sourceFlavors = tagsAndFlavors.getSrcFlavors();
128 String smilFlavorSubType = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
129 SMIL_FLAVOR_SUBTYPE_PROPERTY));
130 String smilTargetFlavorString = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
131 SMIL_TARGET_FLAVOR_PROPERTY));
132 String exportSegmentsDurationString = StringUtils.trimToNull(
133 workflowInstance.getCurrentOperation().getConfiguration(EXPORT_SEGMENTS_DURATION));
134 boolean exportSegmentsDuration = false;
135
136 if (StringUtils.isNotBlank(exportSegmentsDurationString)) {
137 try {
138 exportSegmentsDuration = BooleanUtils.toBoolean(exportSegmentsDurationString);
139 } catch (IllegalArgumentException e) {
140 exportSegmentsDuration = false;
141 logger.warn("Unable to parse {} option value {}. Deactivating export of workflow properties.",
142 EXPORT_SEGMENTS_DURATION, exportSegmentsDurationString);
143 }
144 }
145
146 MediaPackageElementFlavor smilTargetFlavor = null;
147 if (smilTargetFlavorString != null) {
148 smilTargetFlavor = MediaPackageElementFlavor.parseFlavor(smilTargetFlavorString);
149 }
150
151 if (sourceFlavors.isEmpty()) {
152 throw new WorkflowOperationException(String.format("No %s or %s have been specified", SOURCE_FLAVOR_PROPERTY,
153 SOURCE_FLAVORS_PROPERTY));
154 }
155 if (smilFlavorSubType == null && smilTargetFlavor == null) {
156 throw new WorkflowOperationException(String.format("No %s or %s have been specified",
157 SMIL_FLAVOR_SUBTYPE_PROPERTY, SMIL_TARGET_FLAVOR_PROPERTY));
158 }
159 if (sourceFlavors != null && smilTargetFlavor != null) {
160 throw new WorkflowOperationException(String.format("Can't use %s and %s together", SOURCE_FLAVORS_PROPERTY,
161 SMIL_TARGET_FLAVOR_PROPERTY));
162 }
163
164 final String finalSourceFlavors;
165 if (smilTargetFlavor != null) {
166 finalSourceFlavors = sourceFlavors.get(sourceFlavors.size()).toString();
167 } else {
168 finalSourceFlavors = sourceFlavors.stream().map(MediaPackageElementFlavor::toString)
169 .collect(Collectors.joining(","));
170 }
171
172 String referenceTracksFlavor = StringUtils.trimToNull(workflowInstance.getCurrentOperation().getConfiguration(
173 REFERENCE_TRACKS_FLAVOR_PROPERTY));
174 if (referenceTracksFlavor == null) {
175 referenceTracksFlavor = finalSourceFlavors;
176 }
177
178 TrackSelector trackSelector = new TrackSelector();
179 for (String flavor : asList(finalSourceFlavors)) {
180 trackSelector.addFlavor(flavor);
181 }
182 Collection<Track> sourceTracks = trackSelector.select(mp, false);
183 if (sourceTracks.isEmpty()) {
184 logger.info("No source tracks found, skip silence detection");
185 return createResult(mp, Action.SKIP);
186 }
187
188 trackSelector = new TrackSelector();
189 for (String flavor : asList(referenceTracksFlavor)) {
190 trackSelector.addFlavor(flavor);
191 }
192 Collection<Track> referenceTracks = trackSelector.select(mp, false);
193 if (referenceTracks.isEmpty()) {
194
195 throw new WorkflowOperationException(String.format("No tracks found filtered by flavor(s) '%s'",
196 referenceTracksFlavor));
197 }
198 MediaPackageElementBuilder mpeBuilder = MediaPackageElementBuilderFactory.newInstance().newElementBuilder();
199 Map<String, String> exportWorkflowProperties = new HashMap<>();
200 for (Track sourceTrack : sourceTracks) {
201
202 if (!sourceTrack.hasAudio()) {
203 logger.info("Skipping silence detection of track {} since it has no audio", sourceTrack);
204 if (exportSegmentsDuration) {
205 exportWorkflowProperties = exportEmptySegmentDuration(exportWorkflowProperties, sourceTrack);
206 }
207 continue;
208 }
209 logger.info("Executing silence detection on track {}", sourceTrack.getIdentifier());
210 try {
211 Job detectionJob = detetionService.detect(sourceTrack,
212 referenceTracks.toArray(new Track[referenceTracks.size()]));
213 if (!waitForStatus(detectionJob).isSuccess()) {
214 throw new WorkflowOperationException("Silence Detection failed");
215 }
216 Smil smil = smilService.fromXml(detectionJob.getPayload()).getSmil();
217
218 if (smil.getBody().getMediaElements().isEmpty()) {
219 logger.debug("No segments detected in track {}, skip attaching smil file.", sourceTrack.getIdentifier());
220 if (exportSegmentsDuration) {
221 exportWorkflowProperties = exportEmptySegmentDuration(exportWorkflowProperties, sourceTrack);
222 }
223 continue;
224 }
225
226 InputStream is = null;
227 try {
228 is = IOUtils.toInputStream(smil.toXML(), "UTF-8");
229 URI smilURI = workspace.put(mp.getIdentifier().toString(), smil.getId(), TARGET_FILE_NAME, is);
230 MediaPackageElementFlavor smilFlavor = smilTargetFlavor;
231 if (smilFlavor == null) {
232 smilFlavor = new MediaPackageElementFlavor(sourceTrack.getFlavor().getType(), smilFlavorSubType);
233 }
234 Catalog catalog = (Catalog) mpeBuilder.elementFromURI(smilURI, MediaPackageElement.Type.Catalog, smilFlavor);
235 catalog.setIdentifier(smil.getId());
236 mp.add(catalog);
237 } catch (Exception ex) {
238 throw new WorkflowOperationException(String.format(
239 "Failed to put smil into workspace. Silence detection for track %s failed",
240 sourceTrack.getIdentifier()), ex);
241 } finally {
242 IOUtils.closeQuietly(is);
243 }
244 if (exportSegmentsDuration) {
245 long durationMS = 0;
246 for (SmilMediaObject smilElement : smil.getBody().getMediaElements()) {
247 durationMS += getSegmentDurationMS(smilElement);
248 }
249 String durationWfPropertyName = getDurationWfPropertyName(sourceTrack);
250 exportWorkflowProperties.put(durationWfPropertyName,
251 Long.toString(TimeUnit.MILLISECONDS.toSeconds(durationMS)));
252 String relationWfPropertyName = getRelationWfPropertyName(sourceTrack);
253 double durationTrackLengthRelation = 0;
254 if (sourceTrack.getDuration() > 0) {
255 durationTrackLengthRelation = (double)durationMS / (double)sourceTrack.getDuration();
256 durationTrackLengthRelation *= 100;
257 }
258 durationTrackLengthRelation = Math.floor(durationTrackLengthRelation);
259 durationTrackLengthRelation = Math.min(100, durationTrackLengthRelation);
260 durationTrackLengthRelation = Math.max(0, durationTrackLengthRelation);
261 exportWorkflowProperties.put(relationWfPropertyName, String.format("%.0f", durationTrackLengthRelation));
262 }
263 logger.info("Finished silence detection on track {}", sourceTrack.getIdentifier());
264 } catch (SilenceDetectionFailedException ex) {
265 throw new WorkflowOperationException(String.format("Failed to create silence detection job for track %s",
266 sourceTrack.getIdentifier()));
267 } catch (SmilException ex) {
268 throw new WorkflowOperationException(String.format(
269 "Failed to get smil from silence detection job for track %s", sourceTrack.getIdentifier()));
270 }
271 }
272 logger.debug("Finished silence detection workflow operation for media package {}", mp.getIdentifier());
273 return createResult(mp, exportWorkflowProperties, Action.CONTINUE, 0);
274 }
275
276
277
278
279
280
281
282
283 protected long getSegmentDurationMS(SmilMediaObject smilElement) throws SmilException {
284 if (smilElement.isContainer()) {
285 for (SmilMediaObject element : ((SmilMediaContainer) smilElement).getElements()) {
286 return getSegmentDurationMS(element);
287 }
288 }
289 SmilMediaElement smilMediaElement = (SmilMediaElement) smilElement;
290 return smilMediaElement.getClipEndMS() - smilMediaElement.getClipBeginMS();
291 }
292
293
294
295
296
297
298
299 private Map<String, String> exportEmptySegmentDuration(Map<String, String> properties, Track sourceTrack) {
300 String durationWfPropertyName = getDurationWfPropertyName(sourceTrack);
301 properties.put(durationWfPropertyName, Long.toString(0L));
302 String relationWfPropertyName = getRelationWfPropertyName(sourceTrack);
303 properties.put(relationWfPropertyName, String.format("%.0f", 0D));
304 return properties;
305 }
306
307 private String getDurationWfPropertyName(Track sourceTrack) {
308 return String.format("%s_%s_active_audio_duration",
309 sourceTrack.getFlavor().getType(),
310 sourceTrack.getFlavor().getSubtype());
311 }
312
313 private String getRelationWfPropertyName(Track sourceTrack) {
314 return String.format("%s_%s_active_audio_duration_percent",
315 sourceTrack.getFlavor().getType(),
316 sourceTrack.getFlavor().getSubtype());
317 }
318
319 @Override
320 public void activate(ComponentContext cc) {
321 super.activate(cc);
322 logger.info("Registering silence detection workflow operation handler");
323 }
324
325 @Reference
326 public void setDetectionService(SilenceDetectionService detectionService) {
327 this.detetionService = detectionService;
328 }
329
330 @Reference
331 public void setSmilService(SmilService smilService) {
332 this.smilService = smilService;
333 }
334
335 @Reference
336 public void setWorkspace(Workspace workspace) {
337 this.workspace = workspace;
338 }
339
340 @Reference
341 @Override
342 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
343 super.setServiceRegistry(serviceRegistry);
344 }
345
346 }