1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.workflow.handler.composer;
22
23 import static com.entwinemedia.fn.Prelude.chuck;
24 import static com.entwinemedia.fn.Stream.$;
25 import static java.lang.String.format;
26 import static org.opencastproject.util.JobUtil.getPayload;
27
28 import org.opencastproject.composer.api.ComposerService;
29 import org.opencastproject.composer.api.EncoderException;
30 import org.opencastproject.composer.api.EncodingProfile;
31 import org.opencastproject.composer.layout.Dimension;
32 import org.opencastproject.job.api.Job;
33 import org.opencastproject.job.api.JobContext;
34 import org.opencastproject.mediapackage.Attachment;
35 import org.opencastproject.mediapackage.MediaPackage;
36 import org.opencastproject.mediapackage.MediaPackageElement;
37 import org.opencastproject.mediapackage.MediaPackageElement.Type;
38 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
39 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
40 import org.opencastproject.mediapackage.MediaPackageElementParser;
41 import org.opencastproject.mediapackage.MediaPackageException;
42 import org.opencastproject.mediapackage.MediaPackageSupport.Filters;
43 import org.opencastproject.mediapackage.Track;
44 import org.opencastproject.mediapackage.TrackSupport;
45 import org.opencastproject.mediapackage.VideoStream;
46 import org.opencastproject.mediapackage.selector.TrackSelector;
47 import org.opencastproject.serviceregistry.api.ServiceRegistry;
48 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
49 import org.opencastproject.smil.api.util.SmilUtil;
50 import org.opencastproject.util.JobUtil;
51 import org.opencastproject.util.NotFoundException;
52 import org.opencastproject.util.data.Collections;
53 import org.opencastproject.util.data.Tuple;
54 import org.opencastproject.util.data.VCell;
55 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
56 import org.opencastproject.workflow.api.WorkflowInstance;
57 import org.opencastproject.workflow.api.WorkflowOperationException;
58 import org.opencastproject.workflow.api.WorkflowOperationHandler;
59 import org.opencastproject.workflow.api.WorkflowOperationInstance;
60 import org.opencastproject.workflow.api.WorkflowOperationResult;
61 import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
62 import org.opencastproject.workspace.api.Workspace;
63
64 import com.entwinemedia.fn.Fn;
65 import com.entwinemedia.fn.data.Opt;
66
67 import org.apache.commons.io.FilenameUtils;
68 import org.apache.commons.io.IOUtils;
69 import org.apache.commons.lang3.BooleanUtils;
70 import org.apache.commons.lang3.StringUtils;
71 import org.apache.commons.lang3.math.NumberUtils;
72 import org.osgi.service.component.annotations.Component;
73 import org.osgi.service.component.annotations.Reference;
74 import org.slf4j.Logger;
75 import org.slf4j.LoggerFactory;
76 import org.w3c.dom.Node;
77 import org.w3c.dom.NodeList;
78 import org.w3c.dom.smil.SMILDocument;
79 import org.w3c.dom.smil.SMILElement;
80 import org.w3c.dom.smil.SMILMediaElement;
81 import org.w3c.dom.smil.SMILParElement;
82 import org.xml.sax.SAXException;
83
84 import java.io.ByteArrayInputStream;
85 import java.io.File;
86 import java.io.FileInputStream;
87 import java.io.IOException;
88 import java.net.URI;
89 import java.util.ArrayList;
90 import java.util.HashMap;
91 import java.util.List;
92 import java.util.Map;
93 import java.util.Map.Entry;
94 import java.util.UUID;
95
96
97
98
99 @Component(
100 immediate = true,
101 service = WorkflowOperationHandler.class,
102 property = {
103 "service.description=Partial import Workflow Operation Handler",
104 "workflow.operation=partial-import"
105 }
106 )
107 public class PartialImportWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
108
109
110 private static final String SOURCE_PRESENTER_FLAVOR = "source-presenter-flavor";
111 private static final String SOURCE_PRESENTATION_FLAVOR = "source-presentation-flavor";
112 private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
113
114 private static final String TARGET_PRESENTER_FLAVOR = "target-presenter-flavor";
115 private static final String TARGET_PRESENTATION_FLAVOR = "target-presentation-flavor";
116
117 private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
118 private static final String CONCAT_OUTPUT_FRAMERATE = "concat-output-framerate";
119 private static final String TRIM_ENCODING_PROFILE = "trim-encoding-profile";
120 private static final String FORCE_ENCODING_PROFILE = "force-encoding-profile";
121 private static final String PREENCODE_ENCODING_PROFILE = "preencode-encoding-profile";
122
123 private static final String FORCE_ENCODING = "force-encoding";
124 private static final String REQUIRED_EXTENSIONS = "required-extensions";
125 private static final String ENFORCE_DIVISIBLE_BY_TWO = "enforce-divisible-by-two";
126
127
128 private static final Logger logger = LoggerFactory.getLogger(PartialImportWorkflowOperationHandler.class);
129
130
131 private static final String EMPTY_VALUE = "";
132 private static final String NODE_TYPE_AUDIO = "audio";
133 private static final String NODE_TYPE_VIDEO = "video";
134 private static final String FLAVOR_AUDIO_SUFFIX = "-audio";
135 private static final String COLLECTION_ID = "composer";
136 private static final String UNKNOWN_KEY = "unknown";
137 private static final String PRESENTER_KEY = "presenter";
138 private static final String PRESENTATION_KEY = "presentation";
139 private static final String DEFAULT_REQUIRED_EXTENSION = "mp4";
140
141
142 private static final String PREVIEW_PROFILE = "import.preview";
143 private static final String IMAGE_FRAME_PROFILE = "import.image-frame";
144 private static final String SILENT_AUDIO_PROFILE = "import.silent";
145 private static final String IMAGE_MOVIE_PROFILE = "image-movie.work";
146
147
148 private ComposerService composerService = null;
149
150
151 private Workspace workspace = null;
152
153
154
155
156
157
158
159 @Reference
160 public void setComposerService(ComposerService composerService) {
161 this.composerService = composerService;
162 }
163
164
165
166
167
168
169
170
171 @Reference
172 public void setWorkspace(Workspace workspace) {
173 this.workspace = workspace;
174 }
175
176 @Reference
177 @Override
178 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
179 super.setServiceRegistry(serviceRegistry);
180 }
181
182
183
184
185
186
187
188 @Override
189 public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
190 throws WorkflowOperationException {
191 logger.debug("Running partial import workflow operation on workflow {}", workflowInstance.getId());
192
193 List<MediaPackageElement> elementsToClean = new ArrayList<MediaPackageElement>();
194
195 try {
196 return concat(workflowInstance.getMediaPackage(), workflowInstance.getCurrentOperation(), elementsToClean);
197 } catch (Exception e) {
198 throw new WorkflowOperationException(e);
199 } finally {
200 for (MediaPackageElement elem : elementsToClean) {
201 try {
202 workspace.delete(elem.getURI());
203 } catch (Exception e) {
204 logger.warn("Unable to delete element {}", elem, e);
205 }
206 }
207 }
208 }
209
210 private WorkflowOperationResult concat(MediaPackage src, WorkflowOperationInstance operation,
211 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, NotFoundException,
212 MediaPackageException, WorkflowOperationException, ServiceRegistryException {
213 final MediaPackage mediaPackage = (MediaPackage) src.clone();
214 final Long operationId = operation.getId();
215
216
217 final Opt<String> presenterFlavor = getOptConfig(operation, SOURCE_PRESENTER_FLAVOR);
218 final Opt<String> presentationFlavor = getOptConfig(operation, SOURCE_PRESENTATION_FLAVOR);
219 final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(getConfig(operation, SOURCE_SMIL_FLAVOR));
220 final String concatEncodingProfile = getConfig(operation, CONCAT_ENCODING_PROFILE);
221 final Opt<String> concatOutputFramerate = getOptConfig(operation, CONCAT_OUTPUT_FRAMERATE);
222 final String trimEncodingProfile = getConfig(operation, TRIM_ENCODING_PROFILE);
223 final MediaPackageElementFlavor targetPresenterFlavor = parseTargetFlavor(
224 getConfig(operation, TARGET_PRESENTER_FLAVOR), "presenter");
225 final MediaPackageElementFlavor targetPresentationFlavor = parseTargetFlavor(
226 getConfig(operation, TARGET_PRESENTATION_FLAVOR), "presentation");
227 final Opt<EncodingProfile> forceProfile = getForceEncodingProfile(operation);
228 final boolean forceEncoding = BooleanUtils.toBoolean(getOptConfig(operation, FORCE_ENCODING).getOr("false"));
229 final boolean forceDivisible = BooleanUtils.toBoolean(getOptConfig(operation, ENFORCE_DIVISIBLE_BY_TWO).getOr("false"));
230 final List<String> requiredExtensions = getRequiredExtensions(operation);
231 final String preencodeEncodingProfile = getConfig(operation, PREENCODE_ENCODING_PROFILE);
232
233
234
235
236 if (presenterFlavor.isNone() && presentationFlavor.isNone()) {
237 logger.warn("No presenter and presentation flavor has been set.");
238 return createResult(mediaPackage, Action.SKIP);
239 }
240
241 final EncodingProfile preencodeProfile = composerService.getProfile(preencodeEncodingProfile);
242 if (preencodeProfile == null) {
243 throw new WorkflowOperationException("Preencode encoding profile '" + preencodeEncodingProfile + "' was not found");
244 }
245
246 final EncodingProfile concatProfile = composerService.getProfile(concatEncodingProfile);
247 if (concatProfile == null) {
248 throw new WorkflowOperationException("Concat encoding profile '" + concatEncodingProfile + "' was not found");
249 }
250
251 float outputFramerate = -1.0f;
252 if (concatOutputFramerate.isSome()) {
253 if (NumberUtils.isNumber(concatOutputFramerate.get())) {
254 logger.info("Using concat output framerate");
255 outputFramerate = NumberUtils.toFloat(concatOutputFramerate.get());
256 } else {
257 throw new WorkflowOperationException("Unable to parse concat output frame rate!");
258 }
259 }
260
261 final EncodingProfile trimProfile = composerService.getProfile(trimEncodingProfile);
262 if (trimProfile == null) {
263 throw new WorkflowOperationException("Trim encoding profile '" + trimEncodingProfile + "' was not found");
264 }
265
266
267
268 final TrackSelector presenterTrackSelector = mkTrackSelector(presenterFlavor);
269 final TrackSelector presentationTrackSelector = mkTrackSelector(presentationFlavor);
270 List<Track> originalTracks = new ArrayList<Track>();
271
272 for (Track t : presenterTrackSelector.select(mediaPackage, false)) {
273 logger.info("Found partial presenter track {}", t);
274 originalTracks.add(t);
275 }
276
277 for (Track t : presentationTrackSelector.select(mediaPackage, false)) {
278 logger.info("Found partial presentation track {}", t);
279 originalTracks.add(t);
280 }
281
282
283 logger.info("Starting preencoding");
284 originalTracks = preencode(preencodeProfile, originalTracks);
285
286
287
288 final Map<String, Job> jobs = new HashMap<String, Job>();
289
290 final SMILDocument smilDocument;
291 try {
292 smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
293 } catch (SAXException e) {
294 throw new WorkflowOperationException(e);
295 }
296 final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
297 final NodeList sequences = parallel.getTimeChildren();
298 final float trackDurationInSeconds = parallel.getDur();
299 final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
300 for (int i = 0; i < sequences.getLength(); i++) {
301 final SMILElement item = (SMILElement) sequences.item(i);
302
303 for (final String mediaType : new String[] { NODE_TYPE_AUDIO, NODE_TYPE_VIDEO }) {
304 final List<Track> tracks = new ArrayList<Track>();
305 final VCell<String> sourceType = VCell.cell(EMPTY_VALUE);
306
307 final long position = processChildren(0, tracks, item.getChildNodes(), originalTracks, sourceType, mediaType,
308 elementsToClean, operationId);
309
310 if (tracks.isEmpty()) {
311 logger.debug("The tracks list was empty.");
312 continue;
313 }
314 final Track lastTrack = tracks.get(tracks.size() - 1);
315
316 if (position < trackDurationInMs) {
317 final double extendingTime = (trackDurationInMs - position) / 1000d;
318 if (extendingTime > 0) {
319 if (!lastTrack.hasVideo()) {
320 logger.info("Extending {} audio track end by {} seconds with silent audio", sourceType.get(),
321 extendingTime);
322 tracks.add(getSilentAudio(extendingTime, elementsToClean, operationId));
323 } else {
324 logger.info("Extending {} track end with last image frame by {} seconds", sourceType.get(), extendingTime);
325 Attachment tempLastImageFrame = extractLastImageFrame(lastTrack, elementsToClean);
326 tracks.add(createVideoFromImage(tempLastImageFrame, extendingTime, elementsToClean));
327 }
328 }
329 }
330
331 if (tracks.size() < 2) {
332 logger.debug("There were less than 2 tracks, copying track...");
333 if (sourceType.get().startsWith(PRESENTER_KEY)) {
334 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresenterFlavor);
335 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
336 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresentationFlavor);
337 } else {
338 logger.warn("Can't handle unkown source type '{}' for unprocessed track", sourceType.get());
339 }
340 continue;
341 }
342
343 for (final Track t : tracks) {
344 if (!t.hasVideo() && !t.hasAudio()) {
345 logger.error("No audio or video stream available in the track with flavor {}! {}", t.getFlavor(), t);
346 throw new WorkflowOperationException("No audio or video stream available in the track " + t.toString());
347 }
348 }
349
350 if (sourceType.get().startsWith(PRESENTER_KEY)) {
351 logger.info("Concatenating {} track", PRESENTER_KEY);
352 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
353 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
354 logger.info("Concatenating {} track", PRESENTATION_KEY);
355 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
356 } else {
357 logger.warn("Can't handle unknown source type '{}'!", sourceType.get());
358 }
359 }
360 }
361
362
363 if (jobs.size() > 0) {
364 if (!JobUtil.waitForJobs(serviceRegistry, jobs.values()).isSuccess()) {
365 throw new WorkflowOperationException("One of the concat jobs did not complete successfully");
366 }
367 } else {
368 logger.info("No concatenating needed for presenter and presentation tracks, took partial source elements");
369 }
370
371
372 long queueTime = 0L;
373 MediaPackageElementFlavor adjustedTargetPresenterFlavor = targetPresenterFlavor;
374 MediaPackageElementFlavor adjustedTargetPresentationFlavor = targetPresentationFlavor;
375 for (final Entry<String, Job> job : jobs.entrySet()) {
376 final Opt<Job> concatJob = JobUtil.update(serviceRegistry, job.getValue());
377 if (concatJob.isSome()) {
378 final String concatPayload = concatJob.get().getPayload();
379 if (concatPayload != null) {
380 final Track concatTrack;
381 try {
382 concatTrack = (Track) MediaPackageElementParser.getFromXml(concatPayload);
383 } catch (MediaPackageException e) {
384 throw new WorkflowOperationException(e);
385 }
386
387 final String fileName;
388
389
390 if (job.getKey().startsWith(PRESENTER_KEY)) {
391 if (!concatTrack.hasVideo()) {
392 fileName = PRESENTER_KEY.concat(FLAVOR_AUDIO_SUFFIX);
393 adjustedTargetPresenterFlavor = deriveAudioFlavor(targetPresenterFlavor);
394 } else {
395 fileName = PRESENTER_KEY;
396 adjustedTargetPresenterFlavor = targetPresenterFlavor;
397 }
398 concatTrack.setFlavor(adjustedTargetPresenterFlavor);
399 } else if (job.getKey().startsWith(PRESENTATION_KEY)) {
400 if (!concatTrack.hasVideo()) {
401 fileName = PRESENTATION_KEY.concat(FLAVOR_AUDIO_SUFFIX);
402 adjustedTargetPresentationFlavor = deriveAudioFlavor(targetPresentationFlavor);
403 } else {
404 fileName = PRESENTATION_KEY;
405 adjustedTargetPresentationFlavor = targetPresentationFlavor;
406 }
407 concatTrack.setFlavor(adjustedTargetPresentationFlavor);
408 } else {
409 fileName = UNKNOWN_KEY;
410 }
411
412 concatTrack.setURI(workspace.moveTo(concatTrack.getURI(), mediaPackage.getIdentifier().toString(),
413 concatTrack.getIdentifier(),
414 fileName + "." + FilenameUtils.getExtension(concatTrack.getURI().toString())));
415
416 logger.info("Concatenated track {} got flavor '{}'", concatTrack, concatTrack.getFlavor());
417
418 mediaPackage.add(concatTrack);
419 queueTime += concatJob.get().getQueueTime();
420 } else {
421
422 logger.warn("Concat job {} does not contain a payload", concatJob);
423 }
424 } else {
425 logger.warn("Concat job {} could not be updated since it cannot be found", job.getValue());
426 }
427 }
428
429
430 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresentationFlavor, trackDurationInSeconds,
431 elementsToClean);
432 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresentationFlavor),
433 trackDurationInSeconds, elementsToClean);
434 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresenterFlavor, trackDurationInSeconds,
435 elementsToClean);
436 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresenterFlavor),
437 trackDurationInSeconds, elementsToClean);
438
439
440 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, deriveAudioFlavor(targetPresenterFlavor), false, elementsToClean);
441 queueTime += checkForMuxing(mediaPackage, targetPresentationFlavor, deriveAudioFlavor(targetPresentationFlavor), false, elementsToClean);
442
443 adjustAudioTrackTargetFlavor(mediaPackage, targetPresenterFlavor);
444 adjustAudioTrackTargetFlavor(mediaPackage, targetPresentationFlavor);
445
446
447 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, targetPresentationFlavor, false, elementsToClean);
448
449 queueTime += checkForEncodeToStandard(mediaPackage, forceEncoding, forceProfile, requiredExtensions,
450 targetPresenterFlavor, targetPresentationFlavor, elementsToClean);
451
452 final WorkflowOperationResult result = createResult(mediaPackage, Action.CONTINUE, queueTime);
453 logger.debug("Partial import operation completed");
454 return result;
455 }
456
457 protected long checkForEncodeToStandard(MediaPackage mediaPackage, boolean forceEncoding,
458 Opt<EncodingProfile> forceProfile, List<String> requiredExtensions,
459 MediaPackageElementFlavor targetPresenterFlavor, MediaPackageElementFlavor targetPresentationFlavor,
460 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, MediaPackageException,
461 NotFoundException, ServiceRegistryException, WorkflowOperationException {
462 long queueTime = 0;
463 if (forceProfile.isSome()) {
464 Track[] targetPresenterTracks = mediaPackage.getTracks(targetPresenterFlavor);
465 for (Track track : targetPresenterTracks) {
466 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
467 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
468 targetPresenterFlavor, track.getURI(), forceProfile.get());
469 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresenterFlavor, track);
470 elementsToClean.add(track);
471 mediaPackage.remove(track);
472 }
473 }
474
475 if (!targetPresenterFlavor.toString().equalsIgnoreCase(targetPresentationFlavor.toString())) {
476 Track[] targetPresentationTracks = mediaPackage.getTracks(targetPresentationFlavor);
477 for (Track track : targetPresentationTracks) {
478 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
479 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
480 targetPresentationFlavor, track.getURI(), forceProfile.get());
481 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresentationFlavor, track);
482 elementsToClean.add(track);
483 mediaPackage.remove(track);
484 }
485 }
486 }
487 }
488 return queueTime;
489 }
490
491
492
493
494
495
496
497
498
499
500
501 private void createCopyOfTrack(MediaPackage mediaPackage, Track track, MediaPackageElementFlavor targetFlavor)
502 throws IllegalArgumentException, NotFoundException,IOException {
503
504 MediaPackageElementFlavor targetCopyFlavor = null;
505 if (track.hasVideo()) {
506 targetCopyFlavor = targetFlavor;
507 } else {
508 targetCopyFlavor = deriveAudioFlavor(targetFlavor);
509 }
510 logger.debug("Copying track {} with flavor {} using target flavor {}", track.getURI(), track.getFlavor(), targetCopyFlavor);
511 copyPartialToSource(mediaPackage, targetCopyFlavor, track);
512 }
513
514
515
516
517
518
519
520
521
522
523
524
525 private void adjustAudioTrackTargetFlavor(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor)
526 throws IllegalArgumentException, NotFoundException,IOException {
527
528 Track[] targetAudioTracks = mediaPackage.getTracks(deriveAudioFlavor(targetFlavor));
529 for (Track track : targetAudioTracks) {
530 logger.debug("Adding {} to finished audio tracks.", track.getURI());
531 mediaPackage.remove(track);
532 track.setFlavor(targetFlavor);
533 mediaPackage.add(track);
534 }
535 }
536
537 private TrackSelector mkTrackSelector(Opt<String> flavor) throws WorkflowOperationException {
538 final TrackSelector s = new TrackSelector();
539 for (String fs : flavor) {
540 try {
541 final MediaPackageElementFlavor f = MediaPackageElementFlavor.parseFlavor(fs);
542 s.addFlavor(f);
543 s.addFlavor(deriveAudioFlavor(f));
544 } catch (IllegalArgumentException e) {
545 throw new WorkflowOperationException("Flavor '" + fs + "' is malformed");
546 }
547 }
548 return s;
549 }
550
551
552
553
554
555
556
557
558
559
560
561 protected Job startConcatJob(EncodingProfile profile, List<Track> tracks, float outputFramerate, boolean forceDivisible)
562 throws MediaPackageException, EncoderException {
563 final Dimension dim = determineDimension(tracks, forceDivisible);
564 if (outputFramerate > 0.0) {
565 return composerService.concat(profile.getIdentifier(), dim, outputFramerate, true, Collections.toArray(Track.class, tracks));
566 } else {
567 return composerService.concat(profile.getIdentifier(), dim, true, Collections.toArray(Track.class, tracks)); }
568 }
569
570
571
572
573
574
575
576 protected static boolean trackNeedsTobeEncodedToStandard(Track track, List<String> requiredExtensions) {
577 String extension = FilenameUtils.getExtension(track.getURI().toString());
578 for (String requiredExtension : requiredExtensions) {
579 if (requiredExtension.equalsIgnoreCase(extension)) {
580 return false;
581 }
582 }
583 return true;
584 }
585
586
587
588
589
590
591
592
593 protected List<String> getRequiredExtensions(WorkflowOperationInstance operation) {
594 List<String> requiredExtensions = new ArrayList<String>();
595 String configExtensions = null;
596 try {
597 configExtensions = StringUtils.trimToNull(getConfig(operation, REQUIRED_EXTENSIONS));
598 } catch (WorkflowOperationException e) {
599 logger.info(
600 "Required extensions configuration key not specified so will be using default '{}'. Any input file not matching this extension will be re-encoded.",
601 DEFAULT_REQUIRED_EXTENSION);
602 }
603 if (configExtensions != null) {
604 String[] extensions = configExtensions.split(",");
605 for (String extension : extensions) {
606 requiredExtensions.add(extension);
607 }
608 }
609 if (requiredExtensions.size() == 0) {
610 requiredExtensions.add(DEFAULT_REQUIRED_EXTENSION);
611 }
612 return requiredExtensions;
613 }
614
615
616
617
618
619
620
621
622 protected Opt<EncodingProfile> getForceEncodingProfile(WorkflowOperationInstance woi)
623 throws WorkflowOperationException {
624 return getOptConfig(woi, FORCE_ENCODING_PROFILE).map(new Fn<String, EncodingProfile>() {
625 @Override
626 public EncodingProfile apply(String profileName) {
627 for (EncodingProfile profile : Opt.nul(composerService.getProfile(profileName))) {
628 return profile;
629 }
630 return chuck(new WorkflowOperationException("Force encoding profile '" + profileName + "' was not found"));
631 }
632 }).orError(new WorkflowOperationException("Force encoding profile must be set!"));
633 }
634
635
636
637
638
639 private MediaPackageElementFlavor parseTargetFlavor(String flavor, String flavorType)
640 throws WorkflowOperationException {
641 final MediaPackageElementFlavor targetFlavor;
642 try {
643 targetFlavor = MediaPackageElementFlavor.parseFlavor(flavor);
644 if ("*".equals(targetFlavor.getType()) || "*".equals(targetFlavor.getSubtype())) {
645 throw new WorkflowOperationException(format(
646 "Target %s flavor must have a type and a subtype, '*' are not allowed!", flavorType));
647 }
648 } catch (IllegalArgumentException e) {
649 throw new WorkflowOperationException(format("Target %s flavor '%s' is malformed", flavorType, flavor));
650 }
651 return targetFlavor;
652 }
653
654
655 private MediaPackageElementFlavor deriveAudioFlavor(MediaPackageElementFlavor flavor) {
656 return MediaPackageElementFlavor.flavor(flavor.getType().concat(FLAVOR_AUDIO_SUFFIX), flavor.getSubtype());
657 }
658
659
660
661
662
663
664
665
666
667
668 private Dimension determineDimension(List<Track> tracks, boolean forceDivisible) {
669 Tuple<Track, Dimension> trackDimension = getLargestTrack(tracks);
670 if (trackDimension == null)
671 return null;
672
673 if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
674 Dimension scaledDimension = Dimension.dimension((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
675 .getB().getHeight() / 2) * 2);
676 logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
677 trackDimension.getB(), trackDimension.getA());
678 return scaledDimension;
679 } else {
680 logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
681 return trackDimension.getB();
682 }
683 }
684
685
686
687
688
689
690
691
692 private Tuple<Track, Dimension> getLargestTrack(List<Track> tracks) {
693 Track track = null;
694 Dimension dimension = null;
695 for (Track t : tracks) {
696 if (!t.hasVideo())
697 continue;
698
699 VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
700 int frameWidth = videoStreams[0].getFrameWidth();
701 int frameHeight = videoStreams[0].getFrameHeight();
702 if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
703 dimension = Dimension.dimension(frameWidth, frameHeight);
704 track = t;
705 }
706 }
707 if (track == null || dimension == null)
708 return null;
709
710 return Tuple.tuple(track, dimension);
711 }
712
713 private long checkForTrimming(MediaPackage mediaPackage, EncodingProfile trimProfile,
714 MediaPackageElementFlavor targetFlavor, Float videoDuration, List<MediaPackageElement> elementsToClean)
715 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
716 ServiceRegistryException, IOException {
717 MediaPackageElement[] elements = mediaPackage.getElementsByFlavor(targetFlavor);
718 if (elements.length == 0)
719 return 0;
720
721 Track trackToTrim = (Track) elements[0];
722 if (elements.length == 1 && trackToTrim.getDuration() / 1000 > videoDuration) {
723 Long trimSeconds = (long) (trackToTrim.getDuration() / 1000 - videoDuration);
724 logger.info("Shorten track {} to target duration {} by {} seconds",
725 trackToTrim.toString(), videoDuration.toString(), trimSeconds.toString());
726 return trimEnd(mediaPackage, trimProfile, trackToTrim, videoDuration, elementsToClean);
727 } else if (elements.length > 1) {
728 logger.warn("Multiple tracks with flavor {} found! Trimming not possible!", targetFlavor);
729 }
730 return 0;
731 }
732
733 private List<Track> getPureVideoTracks(MediaPackage mediaPackage, MediaPackageElementFlavor videoFlavor) {
734 return $(mediaPackage.getTracks()).filter(Filters.matchesFlavor(videoFlavor).toFn())
735 .filter(Filters.hasVideo.toFn()).filter(Filters.hasNoAudio.toFn()).toList();
736 }
737
738 private List<Track> getPureAudioTracks(MediaPackage mediaPackage, MediaPackageElementFlavor audioFlavor) {
739 return $(mediaPackage.getTracks()).filter(Filters.matchesFlavor(audioFlavor).toFn())
740 .filter(Filters.hasAudio.toFn()).filter(Filters.hasNoVideo.toFn()).toList();
741 }
742
743 protected long checkForMuxing(MediaPackage mediaPackage, MediaPackageElementFlavor targetPresentationFlavor,
744 MediaPackageElementFlavor targetPresenterFlavor, boolean useSuffix, List<MediaPackageElement> elementsToClean)
745 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
746 ServiceRegistryException, IOException {
747
748 long queueTime = 0L;
749
750 List<Track> videoElements = getPureVideoTracks(mediaPackage, targetPresentationFlavor);
751 List<Track> audioElements;
752 if (useSuffix) {
753 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresentationFlavor));
754 } else {
755 audioElements = getPureAudioTracks(mediaPackage, targetPresentationFlavor);
756 }
757
758 Track videoTrack = null;
759 Track audioTrack = null;
760
761 if (videoElements.size() == 1 && audioElements.size() == 0) {
762 videoTrack = videoElements.get(0);
763 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
764 audioTrack = audioElements.get(0);
765 }
766
767 videoElements = getPureVideoTracks(mediaPackage, targetPresenterFlavor);
768 if (useSuffix) {
769 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresenterFlavor));
770 } else {
771 audioElements = getPureAudioTracks(mediaPackage, targetPresenterFlavor);
772 }
773
774 if (videoElements.size() == 1 && audioElements.size() == 0) {
775 videoTrack = videoElements.get(0);
776 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
777 audioTrack = audioElements.get(0);
778 }
779
780 logger.debug("Check for mux between '{}' and '{}' flavors and found video track '{}' and audio track '{}'",
781 targetPresentationFlavor, targetPresenterFlavor, videoTrack, audioTrack);
782 if (videoTrack != null && audioTrack != null) {
783 queueTime += mux(mediaPackage, videoTrack, audioTrack, elementsToClean);
784 return queueTime;
785 } else {
786 return queueTime;
787 }
788 }
789
790
791
792
793
794
795
796 protected long mux(MediaPackage mediaPackage, Track video, Track audio, List<MediaPackageElement> elementsToClean)
797 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
798 ServiceRegistryException, IOException {
799 logger.debug("Muxing video {} and audio {}", video.getURI(), audio.getURI());
800 Job muxJob = composerService.mux(video, audio, PrepareAVWorkflowOperationHandler.MUX_AV_PROFILE);
801 if (!waitForStatus(muxJob).isSuccess()) {
802 throw new WorkflowOperationException("Muxing of audio " + audio + " and video " + video + " failed");
803 }
804 muxJob = serviceRegistry.getJob(muxJob.getId());
805
806 final Track muxed = (Track) MediaPackageElementParser.getFromXml(muxJob.getPayload());
807 if (muxed == null) {
808 throw new WorkflowOperationException("Muxed job " + muxJob + " returned no payload!");
809 }
810 muxed.setFlavor(video.getFlavor());
811 muxed.setURI(workspace.moveTo(muxed.getURI(), mediaPackage.getIdentifier().toString(), muxed.getIdentifier(),
812 FilenameUtils.getName(video.getURI().toString())));
813 elementsToClean.add(audio);
814 mediaPackage.remove(audio);
815 elementsToClean.add(video);
816 mediaPackage.remove(video);
817 mediaPackage.add(muxed);
818 return muxJob.getQueueTime();
819 }
820
821 private void copyPartialToSource(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor, Track track)
822 throws NotFoundException, IOException {
823 FileInputStream in = null;
824 try {
825 Track copyTrack = (Track) track.clone();
826 File originalFile = workspace.get(copyTrack.getURI());
827 in = new FileInputStream(originalFile);
828
829 String elementID = UUID.randomUUID().toString();
830 copyTrack.setURI(workspace.put(mediaPackage.getIdentifier().toString(), elementID,
831 FilenameUtils.getName(copyTrack.getURI().toString()), in));
832 copyTrack.setFlavor(targetFlavor);
833 copyTrack.setIdentifier(elementID);
834 copyTrack.referTo(track);
835 mediaPackage.add(copyTrack);
836 logger.info("Copied partial source element {} to {} with target flavor {}", track.toString(),
837 copyTrack.toString(), targetFlavor.toString());
838 } finally {
839 IOUtils.closeQuietly(in);
840 }
841 }
842
843
844
845
846
847
848
849
850 private List<Track> preencode(EncodingProfile profile, List<Track> tracks)
851 throws MediaPackageException, EncoderException, WorkflowOperationException, NotFoundException,
852 ServiceRegistryException {
853 List<Track> encodedTracks = new ArrayList<>();
854 for (Track track : tracks) {
855 logger.info("Preencoding track {}", track.getIdentifier());
856 Job encodeJob = composerService.encode(track, profile.getIdentifier());
857 if (!waitForStatus(encodeJob).isSuccess()) {
858 throw new WorkflowOperationException("Encoding of track " + track + " failed");
859 }
860 encodeJob = serviceRegistry.getJob(encodeJob.getId());
861 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
862 if (encodedTrack == null) {
863 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
864 }
865 encodedTrack.setIdentifier(track.getIdentifier());
866 encodedTrack.setFlavor(track.getFlavor());
867 encodedTracks.add(encodedTrack);
868 }
869
870 return encodedTracks;
871 }
872
873
874
875
876
877
878
879 private long encodeToStandard(MediaPackage mp, EncodingProfile profile, MediaPackageElementFlavor targetFlavor,
880 Track track) throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
881 ServiceRegistryException, IOException {
882 Job encodeJob = composerService.encode(track, profile.getIdentifier());
883 if (!waitForStatus(encodeJob).isSuccess()) {
884 throw new WorkflowOperationException("Encoding of track " + track + " failed");
885 }
886 encodeJob = serviceRegistry.getJob(encodeJob.getId());
887 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
888 if (encodedTrack == null) {
889 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
890 }
891 URI uri;
892 if (FilenameUtils.getExtension(encodedTrack.getURI().toString()).equalsIgnoreCase(
893 FilenameUtils.getExtension(track.getURI().toString()))) {
894 uri = workspace.moveTo(encodedTrack.getURI(), mp.getIdentifier().toString(), encodedTrack.getIdentifier(),
895 FilenameUtils.getName(track.getURI().toString()));
896 } else {
897
898 uri = workspace.moveTo(
899 encodedTrack.getURI(),
900 mp.getIdentifier().toString(),
901 encodedTrack.getIdentifier(),
902 FilenameUtils.getBaseName(track.getURI().toString()) + "."
903 + FilenameUtils.getExtension(encodedTrack.getURI().toString()));
904 }
905 encodedTrack.setURI(uri);
906 encodedTrack.setFlavor(targetFlavor);
907 mp.add(encodedTrack);
908 return encodeJob.getQueueTime();
909 }
910
911 private long trimEnd(MediaPackage mediaPackage, EncodingProfile trimProfile, Track track, double duration,
912 List<MediaPackageElement> elementsToClean) throws EncoderException, MediaPackageException,
913 WorkflowOperationException, NotFoundException, ServiceRegistryException, IOException {
914 Job trimJob = composerService.trim(track, trimProfile.getIdentifier(), 0, (long) (duration * 1000));
915 if (!waitForStatus(trimJob).isSuccess())
916 throw new WorkflowOperationException("Trimming of track " + track + " failed");
917
918 trimJob = serviceRegistry.getJob(trimJob.getId());
919
920 Track trimmedTrack = (Track) MediaPackageElementParser.getFromXml(trimJob.getPayload());
921 if (trimmedTrack == null)
922 throw new WorkflowOperationException("Trimming track " + track + " failed to produce a track");
923
924 URI uri = workspace.moveTo(trimmedTrack.getURI(), mediaPackage.getIdentifier().toString(),
925 trimmedTrack.getIdentifier(), FilenameUtils.getName(track.getURI().toString()));
926 trimmedTrack.setURI(uri);
927 trimmedTrack.setFlavor(track.getFlavor());
928
929 elementsToClean.add(track);
930 mediaPackage.remove(track);
931 mediaPackage.add(trimmedTrack);
932
933 return trimJob.getQueueTime();
934 }
935
936 private long processChildren(long position, List<Track> tracks, NodeList children, List<Track> originalTracks,
937 VCell<String> type, String mediaType, List<MediaPackageElement> elementsToClean, Long operationId)
938 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException, IOException {
939 for (int j = 0; j < children.getLength(); j++) {
940 Node item = children.item(j);
941 if (item.hasChildNodes()) {
942 position = processChildren(position, tracks, item.getChildNodes(), originalTracks, type, mediaType,
943 elementsToClean, operationId);
944 } else {
945 SMILMediaElement e = (SMILMediaElement) item;
946 if (mediaType.equals(e.getNodeName())) {
947 Track track;
948 try {
949 track = getFromOriginal(e.getId(), originalTracks, type);
950 } catch (IllegalStateException exception) {
951 logger.debug("Skipping smil entry, reason: " + exception.getMessage());
952 continue;
953 }
954 double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
955 long beginInMs = Math.round(beginInSeconds * 1000d);
956
957 if (beginInMs > position) {
958 double positionInSeconds = position / 1000d;
959 if (position == 0) {
960 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
961 logger.info("Extending {} audio track start by {} seconds silent audio", type.get(), beginInSeconds);
962 tracks.add(getSilentAudio(beginInSeconds, elementsToClean, operationId));
963 } else {
964 logger.info("Extending {} track start image frame by {} seconds", type.get(), beginInSeconds);
965 Attachment tempFirstImageFrame = extractImage(track, 0, elementsToClean);
966 tracks.add(createVideoFromImage(tempFirstImageFrame, beginInSeconds, elementsToClean));
967 }
968 position += beginInMs;
969 } else {
970 double fillTime = (beginInMs - position) / 1000d;
971 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
972 logger.info("Fill {} audio track gap from {} to {} with silent audio", type.get(),
973 Double.toString(positionInSeconds), Double.toString(beginInSeconds));
974 tracks.add(getSilentAudio(fillTime, elementsToClean, operationId));
975 } else {
976 logger.info("Fill {} track gap from {} to {} with image frame",
977 type.get(), Double.toString(positionInSeconds), Double.toString(beginInSeconds));
978 Track previousTrack = tracks.get(tracks.size() - 1);
979 Attachment tempLastImageFrame = extractLastImageFrame(previousTrack, elementsToClean);
980 tracks.add(createVideoFromImage(tempLastImageFrame, fillTime, elementsToClean));
981 }
982 position = beginInMs;
983 }
984 }
985 tracks.add(track);
986 position += Math.round(e.getDur() * 1000f);
987 }
988 }
989 }
990 return position;
991 }
992
993 private Track getFromOriginal(String trackId, List<Track> originalTracks, VCell<String> type) {
994 for (Track t : originalTracks) {
995 if (t.getIdentifier().contains(trackId)) {
996 logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
997 if (EMPTY_VALUE.equals(type.get())) {
998 String suffix = (t.hasAudio() && !t.hasVideo()) ? FLAVOR_AUDIO_SUFFIX : "";
999 type.set(t.getFlavor().getType() + suffix);
1000 }
1001 originalTracks.remove(t);
1002 return t;
1003 }
1004 }
1005 throw new IllegalStateException("No track matching smil Track-id: " + trackId);
1006 }
1007
1008 private Track getSilentAudio(final double time, final List<MediaPackageElement> elementsToClean,
1009 final Long operationId) throws EncoderException, MediaPackageException, WorkflowOperationException,
1010 NotFoundException, IOException {
1011 final URI uri = workspace.putInCollection(COLLECTION_ID, operationId + "-silent", new ByteArrayInputStream(
1012 EMPTY_VALUE.getBytes()));
1013 final Attachment emptyAttachment = (Attachment) MediaPackageElementBuilderFactory.newInstance().newElementBuilder()
1014 .elementFromURI(uri, Type.Attachment, MediaPackageElementFlavor.parseFlavor("audio/silent"));
1015 elementsToClean.add(emptyAttachment);
1016
1017 final Job silentAudioJob = composerService.imageToVideo(emptyAttachment, SILENT_AUDIO_PROFILE, time);
1018 if (!waitForStatus(silentAudioJob).isSuccess())
1019 throw new WorkflowOperationException("Silent audio job did not complete successfully");
1020
1021
1022 try {
1023 for (final String payload : getPayload(serviceRegistry, silentAudioJob)) {
1024 final Track silentAudio = (Track) MediaPackageElementParser.getFromXml(payload);
1025 elementsToClean.add(silentAudio);
1026 return silentAudio;
1027 }
1028
1029 throw new WorkflowOperationException(format("Job %s has no payload or cannot be updated", silentAudioJob));
1030 } catch (ServiceRegistryException ex) {
1031 throw new WorkflowOperationException(ex);
1032 }
1033 }
1034
1035 private Track createVideoFromImage(Attachment image, double time, List<MediaPackageElement> elementsToClean)
1036 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1037 Job imageToVideoJob = composerService.imageToVideo(image, IMAGE_MOVIE_PROFILE, time);
1038 if (!waitForStatus(imageToVideoJob).isSuccess())
1039 throw new WorkflowOperationException("Image to video job did not complete successfully");
1040
1041
1042 try {
1043 imageToVideoJob = serviceRegistry.getJob(imageToVideoJob.getId());
1044 } catch (ServiceRegistryException e) {
1045 throw new WorkflowOperationException(e);
1046 }
1047 Track imageVideo = (Track) MediaPackageElementParser.getFromXml(imageToVideoJob.getPayload());
1048 elementsToClean.add(imageVideo);
1049 return imageVideo;
1050 }
1051
1052 private Attachment extractImage(Track presentationTrack, double time, List<MediaPackageElement> elementsToClean)
1053 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1054 Job extractImageJob = composerService.image(presentationTrack, PREVIEW_PROFILE, time);
1055 if (!waitForStatus(extractImageJob).isSuccess())
1056 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1057
1058
1059 try {
1060 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1061 } catch (ServiceRegistryException e) {
1062 throw new WorkflowOperationException(e);
1063 }
1064 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1065 .get(0);
1066 elementsToClean.add(composedImages);
1067 return composedImages;
1068 }
1069
1070 private Attachment extractLastImageFrame(Track presentationTrack, List<MediaPackageElement> elementsToClean)
1071 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1072
1073 Map<String, String> properties = new HashMap<String, String>();
1074
1075 Job extractImageJob = composerService.image(presentationTrack, IMAGE_FRAME_PROFILE, properties);
1076 if (!waitForStatus(extractImageJob).isSuccess())
1077 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1078
1079
1080 try {
1081 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1082 } catch (ServiceRegistryException e) {
1083 throw new WorkflowOperationException(e);
1084 }
1085 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1086 .get(0);
1087 elementsToClean.add(composedImages);
1088 return composedImages;
1089 }
1090 }