View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.workflow.handler.composer;
22  
23  import static java.lang.String.format;
24  import static org.opencastproject.util.JobUtil.getPayload;
25  
26  import org.opencastproject.composer.api.ComposerService;
27  import org.opencastproject.composer.api.EncoderException;
28  import org.opencastproject.composer.api.EncodingProfile;
29  import org.opencastproject.composer.layout.Dimension;
30  import org.opencastproject.job.api.Job;
31  import org.opencastproject.job.api.JobContext;
32  import org.opencastproject.mediapackage.Attachment;
33  import org.opencastproject.mediapackage.MediaPackage;
34  import org.opencastproject.mediapackage.MediaPackageElement;
35  import org.opencastproject.mediapackage.MediaPackageElement.Type;
36  import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
37  import org.opencastproject.mediapackage.MediaPackageElementFlavor;
38  import org.opencastproject.mediapackage.MediaPackageElementParser;
39  import org.opencastproject.mediapackage.MediaPackageException;
40  import org.opencastproject.mediapackage.Track;
41  import org.opencastproject.mediapackage.TrackSupport;
42  import org.opencastproject.mediapackage.VideoStream;
43  import org.opencastproject.mediapackage.selector.TrackSelector;
44  import org.opencastproject.serviceregistry.api.ServiceRegistry;
45  import org.opencastproject.serviceregistry.api.ServiceRegistryException;
46  import org.opencastproject.smil.api.util.SmilUtil;
47  import org.opencastproject.util.JobUtil;
48  import org.opencastproject.util.NotFoundException;
49  import org.opencastproject.util.data.Collections;
50  import org.opencastproject.util.data.Tuple;
51  import org.opencastproject.util.data.VCell;
52  import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
53  import org.opencastproject.workflow.api.WorkflowInstance;
54  import org.opencastproject.workflow.api.WorkflowOperationException;
55  import org.opencastproject.workflow.api.WorkflowOperationHandler;
56  import org.opencastproject.workflow.api.WorkflowOperationInstance;
57  import org.opencastproject.workflow.api.WorkflowOperationResult;
58  import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
59  import org.opencastproject.workspace.api.Workspace;
60  
61  import org.apache.commons.io.FilenameUtils;
62  import org.apache.commons.io.IOUtils;
63  import org.apache.commons.lang3.BooleanUtils;
64  import org.apache.commons.lang3.StringUtils;
65  import org.apache.commons.lang3.math.NumberUtils;
66  import org.osgi.service.component.annotations.Component;
67  import org.osgi.service.component.annotations.Reference;
68  import org.slf4j.Logger;
69  import org.slf4j.LoggerFactory;
70  import org.w3c.dom.Node;
71  import org.w3c.dom.NodeList;
72  import org.w3c.dom.smil.SMILDocument;
73  import org.w3c.dom.smil.SMILElement;
74  import org.w3c.dom.smil.SMILMediaElement;
75  import org.w3c.dom.smil.SMILParElement;
76  import org.xml.sax.SAXException;
77  
78  import java.io.ByteArrayInputStream;
79  import java.io.File;
80  import java.io.FileInputStream;
81  import java.io.IOException;
82  import java.net.URI;
83  import java.util.ArrayList;
84  import java.util.Arrays;
85  import java.util.HashMap;
86  import java.util.List;
87  import java.util.Map;
88  import java.util.Map.Entry;
89  import java.util.Optional;
90  import java.util.stream.Collectors;
91  
92  /**
93   * The workflow definition for handling partial import operations
94   */
95  @Component(
96      immediate = true,
97      service = WorkflowOperationHandler.class,
98      property = {
99          "service.description=Partial import Workflow Operation Handler",
100         "workflow.operation=partial-import"
101     }
102 )
103 public class PartialImportWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
104 
105   /** Workflow configuration keys */
106   private static final String SOURCE_PRESENTER_FLAVOR = "source-presenter-flavor";
107   private static final String SOURCE_PRESENTATION_FLAVOR = "source-presentation-flavor";
108   private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
109 
110   private static final String TARGET_PRESENTER_FLAVOR = "target-presenter-flavor";
111   private static final String TARGET_PRESENTATION_FLAVOR = "target-presentation-flavor";
112 
113   private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
114   private static final String CONCAT_OUTPUT_FRAMERATE = "concat-output-framerate";
115   private static final String TRIM_ENCODING_PROFILE = "trim-encoding-profile";
116   private static final String FORCE_ENCODING_PROFILE = "force-encoding-profile";
117   private static final String PREENCODE_ENCODING_PROFILE = "preencode-encoding-profile";
118 
119   private static final String FORCE_ENCODING = "force-encoding";
120   private static final String REQUIRED_EXTENSIONS = "required-extensions";
121   private static final String ENFORCE_DIVISIBLE_BY_TWO = "enforce-divisible-by-two";
122 
123   /** The logging facility */
124   private static final Logger logger = LoggerFactory.getLogger(PartialImportWorkflowOperationHandler.class);
125 
126   /** Other constants */
127   private static final String EMPTY_VALUE = "";
128   private static final String NODE_TYPE_AUDIO = "audio";
129   private static final String NODE_TYPE_VIDEO = "video";
130   private static final String FLAVOR_AUDIO_SUFFIX = "-audio";
131   private static final String COLLECTION_ID = "composer";
132   private static final String UNKNOWN_KEY = "unknown";
133   private static final String PRESENTER_KEY = "presenter";
134   private static final String PRESENTATION_KEY = "presentation";
135   private static final String DEFAULT_REQUIRED_EXTENSION = "mp4";
136 
137   /** Needed encoding profiles */
138   private static final String PREVIEW_PROFILE = "import.preview";
139   private static final String IMAGE_FRAME_PROFILE = "import.image-frame";
140   private static final String SILENT_AUDIO_PROFILE = "import.silent";
141   private static final String IMAGE_MOVIE_PROFILE = "image-movie.work";
142 
143   /** The composer service */
144   private ComposerService composerService = null;
145 
146   /** The local workspace */
147   private Workspace workspace = null;
148 
149   /**
150    * Callback for the OSGi declarative services configuration.
151    *
152    * @param composerService
153    *          the local composer service
154    */
155   @Reference
156   public void setComposerService(ComposerService composerService) {
157     this.composerService = composerService;
158   }
159 
160   /**
161    * Callback for declarative services configuration that will introduce us to the local workspace service.
162    * Implementation assumes that the reference is configured as being static.
163    *
164    * @param workspace
165    *          an instance of the workspace
166    */
167   @Reference
168   public void setWorkspace(Workspace workspace) {
169     this.workspace = workspace;
170   }
171 
172   @Reference
173   @Override
174   public void setServiceRegistry(ServiceRegistry serviceRegistry) {
175     super.setServiceRegistry(serviceRegistry);
176   }
177 
178   /**
179    * {@inheritDoc}
180    *
181    * @see org.opencastproject.workflow.api.WorkflowOperationHandler#start(org.opencastproject.workflow.api.WorkflowInstance,
182    *      JobContext)
183    */
184   @Override
185   public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
186           throws WorkflowOperationException {
187     logger.debug("Running partial import workflow operation on workflow {}", workflowInstance.getId());
188 
189     List<MediaPackageElement> elementsToClean = new ArrayList<MediaPackageElement>();
190 
191     try {
192       return concat(workflowInstance.getMediaPackage(), workflowInstance.getCurrentOperation(), elementsToClean);
193     } catch (Exception e) {
194       throw new WorkflowOperationException(e);
195     } finally {
196       for (MediaPackageElement elem : elementsToClean) {
197         try {
198           workspace.delete(elem.getURI());
199         } catch (Exception e) {
200           logger.warn("Unable to delete element {}", elem, e);
201         }
202       }
203     }
204   }
205 
206   private WorkflowOperationResult concat(MediaPackage src, WorkflowOperationInstance operation,
207           List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, NotFoundException,
208           MediaPackageException, WorkflowOperationException, ServiceRegistryException {
209     final MediaPackage mediaPackage = (MediaPackage) src.clone();
210     final Long operationId = operation.getId();
211     //
212     // read config options
213     final Optional<String> presenterFlavor = getOptConfig(operation, SOURCE_PRESENTER_FLAVOR);
214     final Optional<String> presentationFlavor = getOptConfig(operation, SOURCE_PRESENTATION_FLAVOR);
215     final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(getConfig(operation, SOURCE_SMIL_FLAVOR));
216     final String concatEncodingProfile = getConfig(operation, CONCAT_ENCODING_PROFILE);
217     final Optional<String> concatOutputFramerate = getOptConfig(operation, CONCAT_OUTPUT_FRAMERATE);
218     final String trimEncodingProfile = getConfig(operation, TRIM_ENCODING_PROFILE);
219     final MediaPackageElementFlavor targetPresenterFlavor = parseTargetFlavor(
220             getConfig(operation, TARGET_PRESENTER_FLAVOR), "presenter");
221     final MediaPackageElementFlavor targetPresentationFlavor = parseTargetFlavor(
222             getConfig(operation, TARGET_PRESENTATION_FLAVOR), "presentation");
223     final boolean forceEncoding = BooleanUtils.toBoolean(getOptConfig(operation, FORCE_ENCODING).orElse("false"));
224     final Optional<EncodingProfile> forceProfile = getForceEncodingProfile(operation, forceEncoding);
225     final boolean forceDivisible = BooleanUtils.toBoolean(getOptConfig(operation, ENFORCE_DIVISIBLE_BY_TWO).orElse("false"));
226     final List<String> requiredExtensions = getRequiredExtensions(operation);
227     final String preencodeEncodingProfile = getConfig(operation, PREENCODE_ENCODING_PROFILE);
228 
229     //
230     // further checks on config options
231     // Skip the worklow if no presenter and presentation flavor has been configured
232     if (presenterFlavor.isEmpty() && presentationFlavor.isEmpty()) {
233       logger.warn("No presenter and presentation flavor has been set.");
234       return createResult(mediaPackage, Action.SKIP);
235     }
236 
237     final EncodingProfile preencodeProfile = composerService.getProfile(preencodeEncodingProfile);
238     if (preencodeProfile == null) {
239       throw new WorkflowOperationException("Preencode encoding profile '" + preencodeEncodingProfile + "' was not found");
240     }
241 
242     final EncodingProfile concatProfile = composerService.getProfile(concatEncodingProfile);
243     if (concatProfile == null) {
244       throw new WorkflowOperationException("Concat encoding profile '" + concatEncodingProfile + "' was not found");
245     }
246 
247     float outputFramerate = -1.0f;
248     if (concatOutputFramerate.isPresent()) {
249       if (NumberUtils.isNumber(concatOutputFramerate.get())) {
250         logger.info("Using concat output framerate");
251         outputFramerate = NumberUtils.toFloat(concatOutputFramerate.get());
252       } else {
253         throw new WorkflowOperationException("Unable to parse concat output frame rate!");
254       }
255     }
256 
257     final EncodingProfile trimProfile = composerService.getProfile(trimEncodingProfile);
258     if (trimProfile == null) {
259       throw new WorkflowOperationException("Trim encoding profile '" + trimEncodingProfile + "' was not found");
260     }
261 
262     //
263     // get tracks
264     final TrackSelector presenterTrackSelector = mkTrackSelector(presenterFlavor);
265     final TrackSelector presentationTrackSelector = mkTrackSelector(presentationFlavor);
266     List<Track> originalTracks = new ArrayList<Track>();
267     // Collecting presenter tracks
268     for (Track t : presenterTrackSelector.select(mediaPackage, false)) {
269       logger.info("Found partial presenter track {}", t);
270       originalTracks.add(t);
271     }
272     // Collecting presentation tracks
273     for (Track t : presentationTrackSelector.select(mediaPackage, false)) {
274       logger.info("Found partial presentation track {}", t);
275       originalTracks.add(t);
276     }
277 
278     // Encode all tracks to same format to enable use of ffmpeg concat-demuxer
279     logger.info("Starting preencoding");
280     originalTracks = preencode(preencodeProfile, originalTracks);
281 
282 
283     // flavor_type -> job
284     final Map<String, Job> jobs = new HashMap<String, Job>();
285     // get SMIL catalog
286     final SMILDocument smilDocument;
287     try {
288       smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
289     } catch (SAXException e) {
290       throw new WorkflowOperationException(e);
291     }
292     final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
293     final NodeList sequences = parallel.getTimeChildren();
294     final float trackDurationInSeconds = parallel.getDur();
295     final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
296     for (int i = 0; i < sequences.getLength(); i++) {
297       final SMILElement item = (SMILElement) sequences.item(i);
298 
299       for (final String mediaType : new String[] { NODE_TYPE_AUDIO, NODE_TYPE_VIDEO }) {
300         final List<Track> tracks = new ArrayList<Track>();
301         final VCell<String> sourceType = VCell.cell(EMPTY_VALUE);
302 
303         final long position = processChildren(0, tracks, item.getChildNodes(), originalTracks, sourceType, mediaType,
304                 elementsToClean, operationId);
305 
306         if (tracks.isEmpty()) {
307           logger.debug("The tracks list was empty.");
308           continue;
309         }
310         final Track lastTrack = tracks.get(tracks.size() - 1);
311 
312         if (position < trackDurationInMs) {
313           final double extendingTime = (trackDurationInMs - position) / 1000d;
314           if (extendingTime > 0) {
315             if (!lastTrack.hasVideo()) {
316               logger.info("Extending {} audio track end by {} seconds with silent audio", sourceType.get(),
317                       extendingTime);
318               tracks.add(getSilentAudio(extendingTime, elementsToClean, operationId));
319             } else {
320               logger.info("Extending {} track end with last image frame by {} seconds", sourceType.get(), extendingTime);
321               Attachment tempLastImageFrame = extractLastImageFrame(lastTrack, elementsToClean);
322               tracks.add(createVideoFromImage(tempLastImageFrame, extendingTime, elementsToClean));
323             }
324           }
325         }
326 
327         if (tracks.size() < 2) {
328           logger.debug("There were less than 2 tracks, copying track...");
329           if (sourceType.get().startsWith(PRESENTER_KEY)) {
330             createCopyOfTrack(mediaPackage, tracks.get(0), targetPresenterFlavor);
331           } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
332             createCopyOfTrack(mediaPackage, tracks.get(0), targetPresentationFlavor);
333           } else {
334             logger.warn("Can't handle unkown source type '{}' for unprocessed track", sourceType.get());
335           }
336           continue;
337         }
338 
339         for (final Track t : tracks) {
340           if (!t.hasVideo() && !t.hasAudio()) {
341             logger.error("No audio or video stream available in the track with flavor {}! {}", t.getFlavor(), t);
342             throw new WorkflowOperationException("No audio or video stream available in the track " + t.toString());
343           }
344         }
345 
346         if (sourceType.get().startsWith(PRESENTER_KEY)) {
347           logger.info("Concatenating {} track", PRESENTER_KEY);
348           jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
349         } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
350           logger.info("Concatenating {} track", PRESENTATION_KEY);
351           jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
352         } else {
353           logger.warn("Can't handle unknown source type '{}'!", sourceType.get());
354         }
355       }
356     }
357 
358     // Wait for the jobs to return
359     if (jobs.size() > 0) {
360       if (!JobUtil.waitForJobs(serviceRegistry, jobs.values()).isSuccess()) {
361         throw new WorkflowOperationException("One of the concat jobs did not complete successfully");
362       }
363     } else {
364       logger.info("No concatenating needed for presenter and presentation tracks, took partial source elements");
365     }
366 
367     // All the jobs have passed, let's update the media package
368     long queueTime = 0L;
369     MediaPackageElementFlavor adjustedTargetPresenterFlavor = targetPresenterFlavor;
370     MediaPackageElementFlavor adjustedTargetPresentationFlavor = targetPresentationFlavor;
371     for (final Entry<String, Job> job : jobs.entrySet()) {
372       final Optional<Job> concatJob = JobUtil.update(serviceRegistry, job.getValue());
373       if (concatJob.isPresent()) {
374         final String concatPayload = concatJob.get().getPayload();
375         if (concatPayload != null) {
376           final Track concatTrack;
377           try {
378             concatTrack = (Track) MediaPackageElementParser.getFromXml(concatPayload);
379           } catch (MediaPackageException e) {
380             throw new WorkflowOperationException(e);
381           }
382 
383           final String fileName;
384 
385           // Adjust the target flavor.
386           if (job.getKey().startsWith(PRESENTER_KEY)) {
387             if (!concatTrack.hasVideo()) {
388               fileName = PRESENTER_KEY.concat(FLAVOR_AUDIO_SUFFIX);
389               adjustedTargetPresenterFlavor = deriveAudioFlavor(targetPresenterFlavor);
390             } else {
391               fileName = PRESENTER_KEY;
392               adjustedTargetPresenterFlavor = targetPresenterFlavor;
393             }
394             concatTrack.setFlavor(adjustedTargetPresenterFlavor);
395           } else if (job.getKey().startsWith(PRESENTATION_KEY)) {
396             if (!concatTrack.hasVideo()) {
397               fileName = PRESENTATION_KEY.concat(FLAVOR_AUDIO_SUFFIX);
398               adjustedTargetPresentationFlavor = deriveAudioFlavor(targetPresentationFlavor);
399             } else {
400               fileName = PRESENTATION_KEY;
401               adjustedTargetPresentationFlavor = targetPresentationFlavor;
402             }
403             concatTrack.setFlavor(adjustedTargetPresentationFlavor);
404           } else {
405             fileName = UNKNOWN_KEY;
406           }
407 
408           concatTrack.setURI(workspace.moveTo(concatTrack.getURI(), mediaPackage.getIdentifier().toString(),
409                   concatTrack.getIdentifier(),
410                   fileName + "." + FilenameUtils.getExtension(concatTrack.getURI().toString())));
411 
412           logger.info("Concatenated track {} got flavor '{}'", concatTrack, concatTrack.getFlavor());
413 
414           mediaPackage.add(concatTrack);
415           queueTime += concatJob.get().getQueueTime();
416         } else {
417           // If there is no payload, then the item has not been distributed.
418           logger.warn("Concat job {} does not contain a payload", concatJob);
419         }
420       } else {
421         logger.warn("Concat job {} could not be updated since it cannot be found", job.getValue());
422       }
423     }
424 
425     // Trim presenter and presentation source track if longer than the duration from the SMIL catalog
426     queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresentationFlavor, trackDurationInSeconds,
427             elementsToClean);
428     queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresentationFlavor),
429             trackDurationInSeconds, elementsToClean);
430     queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresenterFlavor, trackDurationInSeconds,
431             elementsToClean);
432     queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresenterFlavor),
433             trackDurationInSeconds, elementsToClean);
434 
435     // New: Mux within presentation and presenter
436     queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, deriveAudioFlavor(targetPresenterFlavor), false, elementsToClean);
437     queueTime += checkForMuxing(mediaPackage, targetPresentationFlavor, deriveAudioFlavor(targetPresentationFlavor), false, elementsToClean);
438 
439     adjustAudioTrackTargetFlavor(mediaPackage, targetPresenterFlavor);
440     adjustAudioTrackTargetFlavor(mediaPackage, targetPresentationFlavor);
441 
442     // Mux between presentation and presenter? Why?
443     queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, targetPresentationFlavor, false, elementsToClean);
444 
445     queueTime += checkForEncodeToStandard(mediaPackage, forceEncoding, forceProfile, requiredExtensions,
446             targetPresenterFlavor, targetPresentationFlavor, elementsToClean);
447 
448     final WorkflowOperationResult result = createResult(mediaPackage, Action.CONTINUE, queueTime);
449     logger.debug("Partial import operation completed");
450     return result;
451   }
452 
453   protected long checkForEncodeToStandard(MediaPackage mediaPackage, boolean forceEncoding,
454           Optional<EncodingProfile> forceProfile, List<String> requiredExtensions,
455           MediaPackageElementFlavor targetPresenterFlavor, MediaPackageElementFlavor targetPresentationFlavor,
456           List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, MediaPackageException,
457           NotFoundException, ServiceRegistryException, WorkflowOperationException {
458     long queueTime = 0;
459     if (forceProfile.isPresent()) {
460       Track[] targetPresenterTracks = mediaPackage.getTracks(targetPresenterFlavor);
461       for (Track track : targetPresenterTracks) {
462         if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
463           logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
464                   targetPresenterFlavor, track.getURI(), forceProfile.get());
465           queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresenterFlavor, track);
466           elementsToClean.add(track);
467           mediaPackage.remove(track);
468         }
469       }
470       // Skip presentation target if it is the same as the presenter one.
471       if (!targetPresenterFlavor.toString().equalsIgnoreCase(targetPresentationFlavor.toString())) {
472         Track[] targetPresentationTracks = mediaPackage.getTracks(targetPresentationFlavor);
473         for (Track track : targetPresentationTracks) {
474           if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
475             logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
476                     targetPresentationFlavor, track.getURI(), forceProfile.get());
477             queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresentationFlavor, track);
478             elementsToClean.add(track);
479             mediaPackage.remove(track);
480           }
481         }
482       }
483     }
484     return queueTime;
485   }
486 
487   /**
488    * This function creates a copy of a given track in the media package
489    *
490    * @param mediaPackage
491    *          The media package being processed.
492    * @param track
493    *          The track we want to create a copy from.
494    * @param targetFlavor
495    *          The target flavor for the copy of the track.
496    */
497   private void createCopyOfTrack(MediaPackage mediaPackage, Track track, MediaPackageElementFlavor targetFlavor)
498              throws IllegalArgumentException, NotFoundException,IOException {
499 
500     MediaPackageElementFlavor targetCopyFlavor = null;
501     if (track.hasVideo()) {
502       targetCopyFlavor = targetFlavor;
503     } else {
504       targetCopyFlavor = deriveAudioFlavor(targetFlavor);
505     }
506     logger.debug("Copying track {} with flavor {} using target flavor {}", track.getURI(), track.getFlavor(), targetCopyFlavor);
507     copyPartialToSource(mediaPackage, targetCopyFlavor, track);
508   }
509 
510   /**
511    * This functions adjusts the target flavor for audio tracks.
512    * While processing audio tracks, an audio suffix is appended to the type of the audio tracks target flavor.
513    * This functions essentially removes that suffix again and therefore ensures that the target flavor of
514    * audio tracks is set correctly.
515    *
516    * @param mediaPackage
517    *          The media package to look for audio tracks.
518    * @param targetFlavor
519    *          The target flavor for the audio tracks.
520    */
521   private void adjustAudioTrackTargetFlavor(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor)
522              throws IllegalArgumentException, NotFoundException,IOException {
523 
524     Track[] targetAudioTracks = mediaPackage.getTracks(deriveAudioFlavor(targetFlavor));
525     for (Track track : targetAudioTracks) {
526       logger.debug("Adding {} to finished audio tracks.", track.getURI());
527       mediaPackage.remove(track);
528       track.setFlavor(targetFlavor);
529       mediaPackage.add(track);
530     }
531   }
532 
533   private TrackSelector mkTrackSelector(Optional<String> flavor) throws WorkflowOperationException {
534     final TrackSelector s = new TrackSelector();
535     if (flavor.isPresent()) {
536       try {
537         final MediaPackageElementFlavor f = MediaPackageElementFlavor.parseFlavor(flavor.get());
538         s.addFlavor(f);
539         s.addFlavor(deriveAudioFlavor(f));
540       } catch (IllegalArgumentException e) {
541         throw new WorkflowOperationException("Flavor '" + flavor.get() + "' is malformed");
542       }
543     }
544     return s;
545   }
546 
547   /**
548    * Start job to concatenate a list of tracks.
549    *
550    * @param profile
551    *          the encoding profile to use
552    * @param tracks
553    *          non empty track list
554    * @param forceDivisible
555    *          Whether to enforce the track's dimension to be divisible by two
556    */
557   protected Job startConcatJob(EncodingProfile profile, List<Track> tracks, float outputFramerate, boolean forceDivisible)
558           throws MediaPackageException, EncoderException {
559     final Dimension dim = determineDimension(tracks, forceDivisible);
560     if (outputFramerate > 0.0) {
561       return composerService.concat(profile.getIdentifier(), dim, outputFramerate, true, Collections.toArray(Track.class, tracks));
562     } else {
563       return composerService.concat(profile.getIdentifier(), dim, true, Collections.toArray(Track.class, tracks));    }
564   }
565 
566   /**
567    * Determines if the extension of a track is non-standard and therefore should be re-encoded.
568    *
569    * @param track
570    *          The track to check the extension on.
571    */
572   protected static boolean trackNeedsTobeEncodedToStandard(Track track, List<String> requiredExtensions) {
573     String extension = FilenameUtils.getExtension(track.getURI().toString());
574     for (String requiredExtension : requiredExtensions) {
575       if (requiredExtension.equalsIgnoreCase(extension)) {
576         return false;
577       }
578     }
579     return true;
580   }
581 
582   /**
583    * Get the extensions from configuration that don't need to be re-encoded.
584    *
585    * @param operation
586    *          The WorkflowOperationInstance to get the configuration from
587    * @return The list of extensions
588    */
589   protected List<String> getRequiredExtensions(WorkflowOperationInstance operation) {
590     List<String> requiredExtensions = new ArrayList<String>();
591     String configExtensions = null;
592     try {
593       configExtensions = StringUtils.trimToNull(getConfig(operation, REQUIRED_EXTENSIONS));
594     } catch (WorkflowOperationException e) {
595       logger.info(
596               "Required extensions configuration key not specified so will be using default '{}'. Any input file not matching this extension will be re-encoded.",
597               DEFAULT_REQUIRED_EXTENSION);
598     }
599     if (configExtensions != null) {
600       String[] extensions = configExtensions.split(",");
601       for (String extension : extensions) {
602         requiredExtensions.add(extension);
603       }
604     }
605     if (requiredExtensions.size() == 0) {
606       requiredExtensions.add(DEFAULT_REQUIRED_EXTENSION);
607     }
608     return requiredExtensions;
609   }
610 
611   /**
612    * Get the force encoding profile from the operations config options.
613    *
614    * @return the encoding profile if option "force-encoding" is true, none otherwise
615    * @throws WorkflowOperationException
616    *           if there is no such encoding profile or if no encoding profile is configured but force-encoding is true
617    */
618   protected Optional<EncodingProfile> getForceEncodingProfile(WorkflowOperationInstance woi, boolean forceEncoding)
619       throws WorkflowOperationException {
620     if (!forceEncoding) {
621       return Optional.empty();
622     }
623 
624     Optional<String> profileNameOpt = getOptConfig(woi, FORCE_ENCODING_PROFILE);
625     if (forceEncoding && profileNameOpt.isEmpty()) {
626       throw new WorkflowOperationException("Force encoding profile must be set!");
627     }
628 
629     String profileName = profileNameOpt.get();
630     EncodingProfile profile = composerService.getProfile(profileName);
631     if (profile == null) {
632       throw new WorkflowOperationException("Force encoding profile '" + profileName + "' was not found");
633     }
634 
635     return Optional.of(profile);
636   }
637 
638   /**
639    * @param flavorType
640    *          either "presenter" or "presentation", just for error messages
641    */
642   private MediaPackageElementFlavor parseTargetFlavor(String flavor, String flavorType)
643           throws WorkflowOperationException {
644     final MediaPackageElementFlavor targetFlavor;
645     try {
646       targetFlavor = MediaPackageElementFlavor.parseFlavor(flavor);
647       if ("*".equals(targetFlavor.getType()) || "*".equals(targetFlavor.getSubtype())) {
648         throw new WorkflowOperationException(format(
649                 "Target %s flavor must have a type and a subtype, '*' are not allowed!", flavorType));
650       }
651     } catch (IllegalArgumentException e) {
652       throw new WorkflowOperationException(format("Target %s flavor '%s' is malformed", flavorType, flavor));
653     }
654     return targetFlavor;
655   }
656 
657   /** Create a derived audio flavor by appending {@link #FLAVOR_AUDIO_SUFFIX} to the flavor type. */
658   private MediaPackageElementFlavor deriveAudioFlavor(MediaPackageElementFlavor flavor) {
659     return MediaPackageElementFlavor.flavor(flavor.getType().concat(FLAVOR_AUDIO_SUFFIX), flavor.getSubtype());
660   }
661 
662   /**
663    * Determine the largest dimension of the given list of tracks
664    *
665    * @param tracks
666    *          the list of tracks
667    * @param forceDivisible
668    *          Whether to enforce the track's dimension to be divisible by two
669    * @return the largest dimension from the list of track
670    */
671   private Dimension determineDimension(List<Track> tracks, boolean forceDivisible) {
672     Tuple<Track, Dimension> trackDimension = getLargestTrack(tracks);
673     if (trackDimension == null)
674       return null;
675 
676     if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
677       Dimension scaledDimension = Dimension.dimension((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
678               .getB().getHeight() / 2) * 2);
679       logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
680               trackDimension.getB(), trackDimension.getA());
681       return scaledDimension;
682     } else {
683       logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
684       return trackDimension.getB();
685     }
686   }
687 
688   /**
689    * Returns the track with the largest resolution from the list of tracks
690    *
691    * @param tracks
692    *          the list of tracks
693    * @return a {@link Tuple} with the largest track and it's dimension
694    */
695   private Tuple<Track, Dimension> getLargestTrack(List<Track> tracks) {
696     Track track = null;
697     Dimension dimension = null;
698     for (Track t : tracks) {
699       if (!t.hasVideo())
700         continue;
701 
702       VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
703       int frameWidth = videoStreams[0].getFrameWidth();
704       int frameHeight = videoStreams[0].getFrameHeight();
705       if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
706         dimension = Dimension.dimension(frameWidth, frameHeight);
707         track = t;
708       }
709     }
710     if (track == null || dimension == null)
711       return null;
712 
713     return Tuple.tuple(track, dimension);
714   }
715 
716   private long checkForTrimming(MediaPackage mediaPackage, EncodingProfile trimProfile,
717           MediaPackageElementFlavor targetFlavor, Float videoDuration, List<MediaPackageElement> elementsToClean)
718           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
719           ServiceRegistryException, IOException {
720     MediaPackageElement[] elements = mediaPackage.getElementsByFlavor(targetFlavor);
721     if (elements.length == 0)
722       return 0;
723 
724     Track trackToTrim = (Track) elements[0];
725     if (elements.length == 1 && trackToTrim.getDuration() / 1000 > videoDuration) {
726       Long trimSeconds = (long) (trackToTrim.getDuration() / 1000 - videoDuration);
727       logger.info("Shorten track {} to target duration {} by {} seconds",
728               trackToTrim.toString(), videoDuration.toString(), trimSeconds.toString());
729       return trimEnd(mediaPackage, trimProfile, trackToTrim, videoDuration, elementsToClean);
730     } else if (elements.length > 1) {
731       logger.warn("Multiple tracks with flavor {} found! Trimming not possible!", targetFlavor);
732     }
733     return 0;
734   }
735 
736   private List<Track> getPureVideoTracks(MediaPackage mediaPackage, MediaPackageElementFlavor videoFlavor) {
737     return Arrays.stream(mediaPackage.getTracks())
738         .filter(track -> track.getFlavor().matches(videoFlavor))
739         .filter(Track::hasVideo)
740         .filter(track -> !track.hasAudio())
741         .collect(Collectors.toList());
742   }
743 
744   private List<Track> getPureAudioTracks(MediaPackage mediaPackage, MediaPackageElementFlavor audioFlavor) {
745     return Arrays.stream(mediaPackage.getTracks())
746         .filter(track -> track.getFlavor().matches(audioFlavor))
747         .filter(Track::hasAudio)
748         .filter(track -> !track.hasVideo())
749         .collect(Collectors.toList());
750   }
751 
752   protected long checkForMuxing(MediaPackage mediaPackage, MediaPackageElementFlavor targetPresentationFlavor,
753           MediaPackageElementFlavor targetPresenterFlavor, boolean useSuffix, List<MediaPackageElement> elementsToClean)
754           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
755           ServiceRegistryException, IOException {
756 
757     long queueTime = 0L;
758 
759     List<Track> videoElements = getPureVideoTracks(mediaPackage, targetPresentationFlavor);
760     List<Track> audioElements;
761     if (useSuffix) {
762       audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresentationFlavor));
763     } else {
764       audioElements = getPureAudioTracks(mediaPackage, targetPresentationFlavor);
765     }
766 
767     Track videoTrack = null;
768     Track audioTrack = null;
769 
770     if (videoElements.size() == 1 && audioElements.size() == 0) {
771       videoTrack = videoElements.get(0);
772     } else if (videoElements.size() == 0 && audioElements.size() == 1) {
773       audioTrack = audioElements.get(0);
774     }
775 
776     videoElements = getPureVideoTracks(mediaPackage, targetPresenterFlavor);
777     if (useSuffix) {
778       audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresenterFlavor));
779     } else {
780       audioElements = getPureAudioTracks(mediaPackage, targetPresenterFlavor);
781     }
782 
783     if (videoElements.size() == 1 && audioElements.size() == 0) {
784       videoTrack = videoElements.get(0);
785     } else if (videoElements.size() == 0 && audioElements.size() == 1) {
786       audioTrack = audioElements.get(0);
787     }
788 
789     logger.debug("Check for mux between '{}' and '{}' flavors and found video track '{}' and audio track '{}'",
790             targetPresentationFlavor, targetPresenterFlavor, videoTrack, audioTrack);
791     if (videoTrack != null && audioTrack != null) {
792       queueTime += mux(mediaPackage, videoTrack, audioTrack, elementsToClean);
793       return queueTime;
794     } else {
795       return queueTime;
796     }
797   }
798 
799   /**
800    * Mux a video and an audio track. Add the result to media package <code>mediaPackage</code> with the same flavor as
801    * the <code>video</code>.
802    *
803    * @return the mux job's queue time
804    */
805   protected long mux(MediaPackage mediaPackage, Track video, Track audio, List<MediaPackageElement> elementsToClean)
806           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
807           ServiceRegistryException, IOException {
808     logger.debug("Muxing video {} and audio {}", video.getURI(), audio.getURI());
809     Job muxJob = composerService.mux(video, audio, PrepareAVWorkflowOperationHandler.MUX_AV_PROFILE);
810     if (!waitForStatus(muxJob).isSuccess()) {
811       throw new WorkflowOperationException("Muxing of audio " + audio + " and video " + video + " failed");
812     }
813     muxJob = serviceRegistry.getJob(muxJob.getId());
814 
815     final Track muxed = (Track) MediaPackageElementParser.getFromXml(muxJob.getPayload());
816     if (muxed == null) {
817       throw new WorkflowOperationException("Muxed job " + muxJob + " returned no payload!");
818     }
819     muxed.setFlavor(video.getFlavor());
820     muxed.setURI(workspace.moveTo(muxed.getURI(), mediaPackage.getIdentifier().toString(), muxed.getIdentifier(),
821             FilenameUtils.getName(video.getURI().toString())));
822     elementsToClean.add(audio);
823     mediaPackage.remove(audio);
824     elementsToClean.add(video);
825     mediaPackage.remove(video);
826     mediaPackage.add(muxed);
827     return muxJob.getQueueTime();
828   }
829 
830   private void copyPartialToSource(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor, Track track)
831           throws NotFoundException, IOException {
832     FileInputStream in = null;
833     try {
834       Track copyTrack = (Track) track.clone();
835       File originalFile = workspace.get(copyTrack.getURI());
836       in = new FileInputStream(originalFile);
837 
838       copyTrack.generateIdentifier();
839       copyTrack.setURI(workspace.put(mediaPackage.getIdentifier().toString(), copyTrack.getIdentifier(),
840               FilenameUtils.getName(copyTrack.getURI().toString()), in));
841       copyTrack.setFlavor(targetFlavor);
842       copyTrack.referTo(track);
843       mediaPackage.add(copyTrack);
844       logger.info("Copied partial source element {} to {} with target flavor {}", track.toString(),
845               copyTrack.toString(), targetFlavor.toString());
846     } finally {
847       IOUtils.closeQuietly(in);
848     }
849   }
850 
851   /**
852    * Encodes a given list of <code>tracks</code> using the encoding profile <code>profile</code>
853    * and returns the encoded tracks.
854    * Makes sure to keep the tracks ID and Flavor so as to not break later operations.
855    *
856    * @return the encoded tracks
857    */
858   private List<Track> preencode(EncodingProfile profile, List<Track> tracks)
859           throws MediaPackageException, EncoderException, WorkflowOperationException, NotFoundException,
860           ServiceRegistryException {
861     List<Track> encodedTracks = new ArrayList<>();
862     for (Track track : tracks) {
863       logger.info("Preencoding track {}", track.getIdentifier());
864       Job encodeJob = composerService.encode(track, profile.getIdentifier());
865       if (!waitForStatus(encodeJob).isSuccess()) {
866         throw new WorkflowOperationException("Encoding of track " + track + " failed");
867       }
868       encodeJob = serviceRegistry.getJob(encodeJob.getId());
869       Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
870       if (encodedTrack == null) {
871         throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
872       }
873       encodedTrack.setIdentifier(track.getIdentifier());
874       encodedTrack.setFlavor(track.getFlavor());
875       encodedTracks.add(encodedTrack);
876     }
877 
878     return encodedTracks;
879   }
880 
881   /**
882    * Encode <code>track</code> using encoding profile <code>profile</code> and add the result to media package
883    * <code>mp</code> under the given <code>targetFlavor</code>.
884    *
885    * @return the encoder job's queue time
886    */
887   private long encodeToStandard(MediaPackage mp, EncodingProfile profile, MediaPackageElementFlavor targetFlavor,
888           Track track) throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
889           ServiceRegistryException, IOException {
890     Job encodeJob = composerService.encode(track, profile.getIdentifier());
891     if (!waitForStatus(encodeJob).isSuccess()) {
892       throw new WorkflowOperationException("Encoding of track " + track + " failed");
893     }
894     encodeJob = serviceRegistry.getJob(encodeJob.getId());
895     Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
896     if (encodedTrack == null) {
897       throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
898     }
899     URI uri;
900     if (FilenameUtils.getExtension(encodedTrack.getURI().toString()).equalsIgnoreCase(
901             FilenameUtils.getExtension(track.getURI().toString()))) {
902       uri = workspace.moveTo(encodedTrack.getURI(), mp.getIdentifier().toString(), encodedTrack.getIdentifier(),
903               FilenameUtils.getName(track.getURI().toString()));
904     } else {
905       // The new encoded file has a different extension.
906       uri = workspace.moveTo(
907               encodedTrack.getURI(),
908               mp.getIdentifier().toString(),
909               encodedTrack.getIdentifier(),
910               FilenameUtils.getBaseName(track.getURI().toString()) + "."
911                       + FilenameUtils.getExtension(encodedTrack.getURI().toString()));
912     }
913     encodedTrack.setURI(uri);
914     encodedTrack.setFlavor(targetFlavor);
915     mp.add(encodedTrack);
916     return encodeJob.getQueueTime();
917   }
918 
919   private long trimEnd(MediaPackage mediaPackage, EncodingProfile trimProfile, Track track, double duration,
920           List<MediaPackageElement> elementsToClean) throws EncoderException, MediaPackageException,
921           WorkflowOperationException, NotFoundException, ServiceRegistryException, IOException {
922     Job trimJob = composerService.trim(track, trimProfile.getIdentifier(), 0, (long) (duration * 1000));
923     if (!waitForStatus(trimJob).isSuccess())
924       throw new WorkflowOperationException("Trimming of track " + track + " failed");
925 
926     trimJob = serviceRegistry.getJob(trimJob.getId());
927 
928     Track trimmedTrack = (Track) MediaPackageElementParser.getFromXml(trimJob.getPayload());
929     if (trimmedTrack == null)
930       throw new WorkflowOperationException("Trimming track " + track + " failed to produce a track");
931 
932     URI uri = workspace.moveTo(trimmedTrack.getURI(), mediaPackage.getIdentifier().toString(),
933             trimmedTrack.getIdentifier(), FilenameUtils.getName(track.getURI().toString()));
934     trimmedTrack.setURI(uri);
935     trimmedTrack.setFlavor(track.getFlavor());
936 
937     elementsToClean.add(track);
938     mediaPackage.remove(track);
939     mediaPackage.add(trimmedTrack);
940 
941     return trimJob.getQueueTime();
942   }
943 
944   private long processChildren(long position, List<Track> tracks, NodeList children, List<Track> originalTracks,
945           VCell<String> type, String mediaType, List<MediaPackageElement> elementsToClean, Long operationId)
946           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException, IOException {
947     for (int j = 0; j < children.getLength(); j++) {
948       Node item = children.item(j);
949       if (item.hasChildNodes()) {
950         position = processChildren(position, tracks, item.getChildNodes(), originalTracks, type, mediaType,
951                 elementsToClean, operationId);
952       } else {
953         SMILMediaElement e = (SMILMediaElement) item;
954         if (mediaType.equals(e.getNodeName())) {
955           Track track;
956           try {
957             track = getFromOriginal(e.getId(), originalTracks, type);
958           } catch (IllegalStateException exception) {
959             logger.debug("Skipping smil entry, reason: " + exception.getMessage());
960             continue;
961           }
962           double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
963           long beginInMs = Math.round(beginInSeconds * 1000d);
964           // Fill out gaps with first or last frame from video
965           if (beginInMs > position) {
966             double positionInSeconds = position / 1000d;
967             if (position == 0) {
968               if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
969                 logger.info("Extending {} audio track start by {} seconds silent audio", type.get(), beginInSeconds);
970                 tracks.add(getSilentAudio(beginInSeconds, elementsToClean, operationId));
971               } else {
972                 logger.info("Extending {} track start image frame by {} seconds", type.get(), beginInSeconds);
973                 Attachment tempFirstImageFrame = extractImage(track, 0, elementsToClean);
974                 tracks.add(createVideoFromImage(tempFirstImageFrame, beginInSeconds, elementsToClean));
975               }
976               position += beginInMs;
977             } else {
978               double fillTime = (beginInMs - position) / 1000d;
979               if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
980                 logger.info("Fill {} audio track gap from {} to {} with silent audio", type.get(),
981                         Double.toString(positionInSeconds), Double.toString(beginInSeconds));
982                 tracks.add(getSilentAudio(fillTime, elementsToClean, operationId));
983               } else {
984                 logger.info("Fill {} track gap from {} to {} with image frame",
985                         type.get(), Double.toString(positionInSeconds), Double.toString(beginInSeconds));
986                 Track previousTrack = tracks.get(tracks.size() - 1);
987                 Attachment tempLastImageFrame = extractLastImageFrame(previousTrack, elementsToClean);
988                 tracks.add(createVideoFromImage(tempLastImageFrame, fillTime, elementsToClean));
989               }
990               position = beginInMs;
991             }
992           }
993           tracks.add(track);
994           position += Math.round(e.getDur() * 1000f);
995         }
996       }
997     }
998     return position;
999   }
1000 
1001   private Track getFromOriginal(String trackId, List<Track> originalTracks, VCell<String> type) {
1002     for (Track t : originalTracks) {
1003       if (t.getIdentifier().contains(trackId)) {
1004         logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
1005         if (EMPTY_VALUE.equals(type.get())) {
1006           String suffix = (t.hasAudio() && !t.hasVideo()) ? FLAVOR_AUDIO_SUFFIX : "";
1007           type.set(t.getFlavor().getType() + suffix);
1008         }
1009         originalTracks.remove(t);
1010         return t;
1011       }
1012     }
1013     throw new IllegalStateException("No track matching smil Track-id: " + trackId);
1014   }
1015 
1016   private Track getSilentAudio(final double time, final List<MediaPackageElement> elementsToClean,
1017           final Long operationId) throws EncoderException, MediaPackageException, WorkflowOperationException,
1018           NotFoundException, IOException {
1019     final URI uri = workspace.putInCollection(COLLECTION_ID, operationId + "-silent", new ByteArrayInputStream(
1020             EMPTY_VALUE.getBytes()));
1021     final Attachment emptyAttachment = (Attachment) MediaPackageElementBuilderFactory.newInstance().newElementBuilder()
1022             .elementFromURI(uri, Type.Attachment, MediaPackageElementFlavor.parseFlavor("audio/silent"));
1023     elementsToClean.add(emptyAttachment);
1024 
1025     final Job silentAudioJob = composerService.imageToVideo(emptyAttachment, SILENT_AUDIO_PROFILE, time);
1026     if (!waitForStatus(silentAudioJob).isSuccess())
1027       throw new WorkflowOperationException("Silent audio job did not complete successfully");
1028 
1029     // Get the latest copy
1030     try {
1031       Optional<String> payloadOpt = getPayload(serviceRegistry, silentAudioJob);
1032       if (payloadOpt.isPresent()) {
1033         final Track silentAudio = (Track) MediaPackageElementParser.getFromXml(payloadOpt.get());
1034         elementsToClean.add(silentAudio);
1035         return silentAudio;
1036       }
1037       // none
1038       throw new WorkflowOperationException(format("Job %s has no payload or cannot be updated", silentAudioJob));
1039     } catch (ServiceRegistryException ex) {
1040       throw new WorkflowOperationException(ex);
1041     }
1042   }
1043 
1044   private Track createVideoFromImage(Attachment image, double time, List<MediaPackageElement> elementsToClean)
1045           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1046     Job imageToVideoJob = composerService.imageToVideo(image, IMAGE_MOVIE_PROFILE, time);
1047     if (!waitForStatus(imageToVideoJob).isSuccess())
1048       throw new WorkflowOperationException("Image to video job did not complete successfully");
1049 
1050     // Get the latest copy
1051     try {
1052       imageToVideoJob = serviceRegistry.getJob(imageToVideoJob.getId());
1053     } catch (ServiceRegistryException e) {
1054       throw new WorkflowOperationException(e);
1055     }
1056     Track imageVideo = (Track) MediaPackageElementParser.getFromXml(imageToVideoJob.getPayload());
1057     elementsToClean.add(imageVideo);
1058     return imageVideo;
1059   }
1060 
1061   private Attachment extractImage(Track presentationTrack, double time, List<MediaPackageElement> elementsToClean)
1062           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1063     Job extractImageJob = composerService.image(presentationTrack, PREVIEW_PROFILE, time);
1064     if (!waitForStatus(extractImageJob).isSuccess())
1065       throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1066 
1067     // Get the latest copy
1068     try {
1069       extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1070     } catch (ServiceRegistryException e) {
1071       throw new WorkflowOperationException(e);
1072     }
1073     Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1074             .get(0);
1075     elementsToClean.add(composedImages);
1076     return composedImages;
1077   }
1078 
1079   private Attachment extractLastImageFrame(Track presentationTrack, List<MediaPackageElement> elementsToClean)
1080           throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1081     // Pass empty properties to the composer service, because the given profile requires none
1082     Map<String, String> properties = new HashMap<String, String>();
1083 
1084     Job extractImageJob = composerService.image(presentationTrack, IMAGE_FRAME_PROFILE, properties);
1085     if (!waitForStatus(extractImageJob).isSuccess())
1086       throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1087 
1088     // Get the latest copy
1089     try {
1090       extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1091     } catch (ServiceRegistryException e) {
1092       throw new WorkflowOperationException(e);
1093     }
1094     Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1095             .get(0);
1096     elementsToClean.add(composedImages);
1097     return composedImages;
1098   }
1099 }