1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.workflow.handler.composer;
22
23 import static java.lang.String.format;
24 import static org.opencastproject.util.JobUtil.getPayload;
25
26 import org.opencastproject.composer.api.ComposerService;
27 import org.opencastproject.composer.api.EncoderException;
28 import org.opencastproject.composer.api.EncodingProfile;
29 import org.opencastproject.composer.layout.Dimension;
30 import org.opencastproject.job.api.Job;
31 import org.opencastproject.job.api.JobContext;
32 import org.opencastproject.mediapackage.Attachment;
33 import org.opencastproject.mediapackage.MediaPackage;
34 import org.opencastproject.mediapackage.MediaPackageElement;
35 import org.opencastproject.mediapackage.MediaPackageElement.Type;
36 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
37 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
38 import org.opencastproject.mediapackage.MediaPackageElementParser;
39 import org.opencastproject.mediapackage.MediaPackageException;
40 import org.opencastproject.mediapackage.Track;
41 import org.opencastproject.mediapackage.TrackSupport;
42 import org.opencastproject.mediapackage.VideoStream;
43 import org.opencastproject.mediapackage.selector.TrackSelector;
44 import org.opencastproject.serviceregistry.api.ServiceRegistry;
45 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
46 import org.opencastproject.smil.api.util.SmilUtil;
47 import org.opencastproject.util.JobUtil;
48 import org.opencastproject.util.NotFoundException;
49 import org.opencastproject.util.data.Collections;
50 import org.opencastproject.util.data.Tuple;
51 import org.opencastproject.util.data.VCell;
52 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
53 import org.opencastproject.workflow.api.WorkflowInstance;
54 import org.opencastproject.workflow.api.WorkflowOperationException;
55 import org.opencastproject.workflow.api.WorkflowOperationHandler;
56 import org.opencastproject.workflow.api.WorkflowOperationInstance;
57 import org.opencastproject.workflow.api.WorkflowOperationResult;
58 import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
59 import org.opencastproject.workspace.api.Workspace;
60
61 import org.apache.commons.io.FilenameUtils;
62 import org.apache.commons.io.IOUtils;
63 import org.apache.commons.lang3.BooleanUtils;
64 import org.apache.commons.lang3.StringUtils;
65 import org.apache.commons.lang3.math.NumberUtils;
66 import org.osgi.service.component.annotations.Component;
67 import org.osgi.service.component.annotations.Reference;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70 import org.w3c.dom.Node;
71 import org.w3c.dom.NodeList;
72 import org.w3c.dom.smil.SMILDocument;
73 import org.w3c.dom.smil.SMILElement;
74 import org.w3c.dom.smil.SMILMediaElement;
75 import org.w3c.dom.smil.SMILParElement;
76 import org.xml.sax.SAXException;
77
78 import java.io.ByteArrayInputStream;
79 import java.io.File;
80 import java.io.FileInputStream;
81 import java.io.IOException;
82 import java.net.URI;
83 import java.util.ArrayList;
84 import java.util.Arrays;
85 import java.util.HashMap;
86 import java.util.List;
87 import java.util.Map;
88 import java.util.Map.Entry;
89 import java.util.Optional;
90 import java.util.stream.Collectors;
91
92
93
94
95 @Component(
96 immediate = true,
97 service = WorkflowOperationHandler.class,
98 property = {
99 "service.description=Partial import Workflow Operation Handler",
100 "workflow.operation=partial-import"
101 }
102 )
103 public class PartialImportWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
104
105
106 private static final String SOURCE_PRESENTER_FLAVOR = "source-presenter-flavor";
107 private static final String SOURCE_PRESENTATION_FLAVOR = "source-presentation-flavor";
108 private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
109
110 private static final String TARGET_PRESENTER_FLAVOR = "target-presenter-flavor";
111 private static final String TARGET_PRESENTATION_FLAVOR = "target-presentation-flavor";
112
113 private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
114 private static final String CONCAT_OUTPUT_FRAMERATE = "concat-output-framerate";
115 private static final String TRIM_ENCODING_PROFILE = "trim-encoding-profile";
116 private static final String FORCE_ENCODING_PROFILE = "force-encoding-profile";
117 private static final String PREENCODE_ENCODING_PROFILE = "preencode-encoding-profile";
118
119 private static final String FORCE_ENCODING = "force-encoding";
120 private static final String REQUIRED_EXTENSIONS = "required-extensions";
121 private static final String ENFORCE_DIVISIBLE_BY_TWO = "enforce-divisible-by-two";
122
123
124 private static final Logger logger = LoggerFactory.getLogger(PartialImportWorkflowOperationHandler.class);
125
126
127 private static final String EMPTY_VALUE = "";
128 private static final String NODE_TYPE_AUDIO = "audio";
129 private static final String NODE_TYPE_VIDEO = "video";
130 private static final String FLAVOR_AUDIO_SUFFIX = "-audio";
131 private static final String COLLECTION_ID = "composer";
132 private static final String UNKNOWN_KEY = "unknown";
133 private static final String PRESENTER_KEY = "presenter";
134 private static final String PRESENTATION_KEY = "presentation";
135 private static final String DEFAULT_REQUIRED_EXTENSION = "mp4";
136
137
138 private static final String PREVIEW_PROFILE = "import.preview";
139 private static final String IMAGE_FRAME_PROFILE = "import.image-frame";
140 private static final String SILENT_AUDIO_PROFILE = "import.silent";
141 private static final String IMAGE_MOVIE_PROFILE = "image-movie.work";
142
143
144 private ComposerService composerService = null;
145
146
147 private Workspace workspace = null;
148
149
150
151
152
153
154
155 @Reference
156 public void setComposerService(ComposerService composerService) {
157 this.composerService = composerService;
158 }
159
160
161
162
163
164
165
166
167 @Reference
168 public void setWorkspace(Workspace workspace) {
169 this.workspace = workspace;
170 }
171
172 @Reference
173 @Override
174 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
175 super.setServiceRegistry(serviceRegistry);
176 }
177
178
179
180
181
182
183
184 @Override
185 public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
186 throws WorkflowOperationException {
187 logger.debug("Running partial import workflow operation on workflow {}", workflowInstance.getId());
188
189 List<MediaPackageElement> elementsToClean = new ArrayList<MediaPackageElement>();
190
191 try {
192 return concat(workflowInstance.getMediaPackage(), workflowInstance.getCurrentOperation(), elementsToClean);
193 } catch (Exception e) {
194 throw new WorkflowOperationException(e);
195 } finally {
196 for (MediaPackageElement elem : elementsToClean) {
197 try {
198 workspace.delete(elem.getURI());
199 } catch (Exception e) {
200 logger.warn("Unable to delete element {}", elem, e);
201 }
202 }
203 }
204 }
205
206 private WorkflowOperationResult concat(MediaPackage src, WorkflowOperationInstance operation,
207 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, NotFoundException,
208 MediaPackageException, WorkflowOperationException, ServiceRegistryException {
209 final MediaPackage mediaPackage = (MediaPackage) src.clone();
210 final Long operationId = operation.getId();
211
212
213 final Optional<String> presenterFlavor = getOptConfig(operation, SOURCE_PRESENTER_FLAVOR);
214 final Optional<String> presentationFlavor = getOptConfig(operation, SOURCE_PRESENTATION_FLAVOR);
215 final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(getConfig(operation, SOURCE_SMIL_FLAVOR));
216 final String concatEncodingProfile = getConfig(operation, CONCAT_ENCODING_PROFILE);
217 final Optional<String> concatOutputFramerate = getOptConfig(operation, CONCAT_OUTPUT_FRAMERATE);
218 final String trimEncodingProfile = getConfig(operation, TRIM_ENCODING_PROFILE);
219 final MediaPackageElementFlavor targetPresenterFlavor = parseTargetFlavor(
220 getConfig(operation, TARGET_PRESENTER_FLAVOR), "presenter");
221 final MediaPackageElementFlavor targetPresentationFlavor = parseTargetFlavor(
222 getConfig(operation, TARGET_PRESENTATION_FLAVOR), "presentation");
223 final boolean forceEncoding = BooleanUtils.toBoolean(getOptConfig(operation, FORCE_ENCODING).orElse("false"));
224 final Optional<EncodingProfile> forceProfile = getForceEncodingProfile(operation, forceEncoding);
225 final boolean forceDivisible = BooleanUtils.toBoolean(getOptConfig(operation, ENFORCE_DIVISIBLE_BY_TWO).orElse("false"));
226 final List<String> requiredExtensions = getRequiredExtensions(operation);
227 final String preencodeEncodingProfile = getConfig(operation, PREENCODE_ENCODING_PROFILE);
228
229
230
231
232 if (presenterFlavor.isEmpty() && presentationFlavor.isEmpty()) {
233 logger.warn("No presenter and presentation flavor has been set.");
234 return createResult(mediaPackage, Action.SKIP);
235 }
236
237 final EncodingProfile preencodeProfile = composerService.getProfile(preencodeEncodingProfile);
238 if (preencodeProfile == null) {
239 throw new WorkflowOperationException("Preencode encoding profile '" + preencodeEncodingProfile + "' was not found");
240 }
241
242 final EncodingProfile concatProfile = composerService.getProfile(concatEncodingProfile);
243 if (concatProfile == null) {
244 throw new WorkflowOperationException("Concat encoding profile '" + concatEncodingProfile + "' was not found");
245 }
246
247 float outputFramerate = -1.0f;
248 if (concatOutputFramerate.isPresent()) {
249 if (NumberUtils.isNumber(concatOutputFramerate.get())) {
250 logger.info("Using concat output framerate");
251 outputFramerate = NumberUtils.toFloat(concatOutputFramerate.get());
252 } else {
253 throw new WorkflowOperationException("Unable to parse concat output frame rate!");
254 }
255 }
256
257 final EncodingProfile trimProfile = composerService.getProfile(trimEncodingProfile);
258 if (trimProfile == null) {
259 throw new WorkflowOperationException("Trim encoding profile '" + trimEncodingProfile + "' was not found");
260 }
261
262
263
264 final TrackSelector presenterTrackSelector = mkTrackSelector(presenterFlavor);
265 final TrackSelector presentationTrackSelector = mkTrackSelector(presentationFlavor);
266 List<Track> originalTracks = new ArrayList<Track>();
267
268 for (Track t : presenterTrackSelector.select(mediaPackage, false)) {
269 logger.info("Found partial presenter track {}", t);
270 originalTracks.add(t);
271 }
272
273 for (Track t : presentationTrackSelector.select(mediaPackage, false)) {
274 logger.info("Found partial presentation track {}", t);
275 originalTracks.add(t);
276 }
277
278
279 logger.info("Starting preencoding");
280 originalTracks = preencode(preencodeProfile, originalTracks);
281
282
283
284 final Map<String, Job> jobs = new HashMap<String, Job>();
285
286 final SMILDocument smilDocument;
287 try {
288 smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
289 } catch (SAXException e) {
290 throw new WorkflowOperationException(e);
291 }
292 final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
293 final NodeList sequences = parallel.getTimeChildren();
294 final float trackDurationInSeconds = parallel.getDur();
295 final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
296 for (int i = 0; i < sequences.getLength(); i++) {
297 final SMILElement item = (SMILElement) sequences.item(i);
298
299 for (final String mediaType : new String[] { NODE_TYPE_AUDIO, NODE_TYPE_VIDEO }) {
300 final List<Track> tracks = new ArrayList<Track>();
301 final VCell<String> sourceType = VCell.cell(EMPTY_VALUE);
302
303 final long position = processChildren(0, tracks, item.getChildNodes(), originalTracks, sourceType, mediaType,
304 elementsToClean, operationId);
305
306 if (tracks.isEmpty()) {
307 logger.debug("The tracks list was empty.");
308 continue;
309 }
310 final Track lastTrack = tracks.get(tracks.size() - 1);
311
312 if (position < trackDurationInMs) {
313 final double extendingTime = (trackDurationInMs - position) / 1000d;
314 if (extendingTime > 0) {
315 if (!lastTrack.hasVideo()) {
316 logger.info("Extending {} audio track end by {} seconds with silent audio", sourceType.get(),
317 extendingTime);
318 tracks.add(getSilentAudio(extendingTime, elementsToClean, operationId));
319 } else {
320 logger.info("Extending {} track end with last image frame by {} seconds", sourceType.get(), extendingTime);
321 Attachment tempLastImageFrame = extractLastImageFrame(lastTrack, elementsToClean);
322 tracks.add(createVideoFromImage(tempLastImageFrame, extendingTime, elementsToClean));
323 }
324 }
325 }
326
327 if (tracks.size() < 2) {
328 logger.debug("There were less than 2 tracks, copying track...");
329 if (sourceType.get().startsWith(PRESENTER_KEY)) {
330 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresenterFlavor);
331 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
332 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresentationFlavor);
333 } else {
334 logger.warn("Can't handle unkown source type '{}' for unprocessed track", sourceType.get());
335 }
336 continue;
337 }
338
339 for (final Track t : tracks) {
340 if (!t.hasVideo() && !t.hasAudio()) {
341 logger.error("No audio or video stream available in the track with flavor {}! {}", t.getFlavor(), t);
342 throw new WorkflowOperationException("No audio or video stream available in the track " + t.toString());
343 }
344 }
345
346 if (sourceType.get().startsWith(PRESENTER_KEY)) {
347 logger.info("Concatenating {} track", PRESENTER_KEY);
348 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
349 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
350 logger.info("Concatenating {} track", PRESENTATION_KEY);
351 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
352 } else {
353 logger.warn("Can't handle unknown source type '{}'!", sourceType.get());
354 }
355 }
356 }
357
358
359 if (jobs.size() > 0) {
360 if (!JobUtil.waitForJobs(serviceRegistry, jobs.values()).isSuccess()) {
361 throw new WorkflowOperationException("One of the concat jobs did not complete successfully");
362 }
363 } else {
364 logger.info("No concatenating needed for presenter and presentation tracks, took partial source elements");
365 }
366
367
368 long queueTime = 0L;
369 MediaPackageElementFlavor adjustedTargetPresenterFlavor = targetPresenterFlavor;
370 MediaPackageElementFlavor adjustedTargetPresentationFlavor = targetPresentationFlavor;
371 for (final Entry<String, Job> job : jobs.entrySet()) {
372 final Optional<Job> concatJob = JobUtil.update(serviceRegistry, job.getValue());
373 if (concatJob.isPresent()) {
374 final String concatPayload = concatJob.get().getPayload();
375 if (concatPayload != null) {
376 final Track concatTrack;
377 try {
378 concatTrack = (Track) MediaPackageElementParser.getFromXml(concatPayload);
379 } catch (MediaPackageException e) {
380 throw new WorkflowOperationException(e);
381 }
382
383 final String fileName;
384
385
386 if (job.getKey().startsWith(PRESENTER_KEY)) {
387 if (!concatTrack.hasVideo()) {
388 fileName = PRESENTER_KEY.concat(FLAVOR_AUDIO_SUFFIX);
389 adjustedTargetPresenterFlavor = deriveAudioFlavor(targetPresenterFlavor);
390 } else {
391 fileName = PRESENTER_KEY;
392 adjustedTargetPresenterFlavor = targetPresenterFlavor;
393 }
394 concatTrack.setFlavor(adjustedTargetPresenterFlavor);
395 } else if (job.getKey().startsWith(PRESENTATION_KEY)) {
396 if (!concatTrack.hasVideo()) {
397 fileName = PRESENTATION_KEY.concat(FLAVOR_AUDIO_SUFFIX);
398 adjustedTargetPresentationFlavor = deriveAudioFlavor(targetPresentationFlavor);
399 } else {
400 fileName = PRESENTATION_KEY;
401 adjustedTargetPresentationFlavor = targetPresentationFlavor;
402 }
403 concatTrack.setFlavor(adjustedTargetPresentationFlavor);
404 } else {
405 fileName = UNKNOWN_KEY;
406 }
407
408 concatTrack.setURI(workspace.moveTo(concatTrack.getURI(), mediaPackage.getIdentifier().toString(),
409 concatTrack.getIdentifier(),
410 fileName + "." + FilenameUtils.getExtension(concatTrack.getURI().toString())));
411
412 logger.info("Concatenated track {} got flavor '{}'", concatTrack, concatTrack.getFlavor());
413
414 mediaPackage.add(concatTrack);
415 queueTime += concatJob.get().getQueueTime();
416 } else {
417
418 logger.warn("Concat job {} does not contain a payload", concatJob);
419 }
420 } else {
421 logger.warn("Concat job {} could not be updated since it cannot be found", job.getValue());
422 }
423 }
424
425
426 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresentationFlavor, trackDurationInSeconds,
427 elementsToClean);
428 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresentationFlavor),
429 trackDurationInSeconds, elementsToClean);
430 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresenterFlavor, trackDurationInSeconds,
431 elementsToClean);
432 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresenterFlavor),
433 trackDurationInSeconds, elementsToClean);
434
435
436 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, deriveAudioFlavor(targetPresenterFlavor), false, elementsToClean);
437 queueTime += checkForMuxing(mediaPackage, targetPresentationFlavor, deriveAudioFlavor(targetPresentationFlavor), false, elementsToClean);
438
439 adjustAudioTrackTargetFlavor(mediaPackage, targetPresenterFlavor);
440 adjustAudioTrackTargetFlavor(mediaPackage, targetPresentationFlavor);
441
442
443 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, targetPresentationFlavor, false, elementsToClean);
444
445 queueTime += checkForEncodeToStandard(mediaPackage, forceEncoding, forceProfile, requiredExtensions,
446 targetPresenterFlavor, targetPresentationFlavor, elementsToClean);
447
448 final WorkflowOperationResult result = createResult(mediaPackage, Action.CONTINUE, queueTime);
449 logger.debug("Partial import operation completed");
450 return result;
451 }
452
453 protected long checkForEncodeToStandard(MediaPackage mediaPackage, boolean forceEncoding,
454 Optional<EncodingProfile> forceProfile, List<String> requiredExtensions,
455 MediaPackageElementFlavor targetPresenterFlavor, MediaPackageElementFlavor targetPresentationFlavor,
456 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, MediaPackageException,
457 NotFoundException, ServiceRegistryException, WorkflowOperationException {
458 long queueTime = 0;
459 if (forceProfile.isPresent()) {
460 Track[] targetPresenterTracks = mediaPackage.getTracks(targetPresenterFlavor);
461 for (Track track : targetPresenterTracks) {
462 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
463 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
464 targetPresenterFlavor, track.getURI(), forceProfile.get());
465 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresenterFlavor, track);
466 elementsToClean.add(track);
467 mediaPackage.remove(track);
468 }
469 }
470
471 if (!targetPresenterFlavor.toString().equalsIgnoreCase(targetPresentationFlavor.toString())) {
472 Track[] targetPresentationTracks = mediaPackage.getTracks(targetPresentationFlavor);
473 for (Track track : targetPresentationTracks) {
474 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
475 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
476 targetPresentationFlavor, track.getURI(), forceProfile.get());
477 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresentationFlavor, track);
478 elementsToClean.add(track);
479 mediaPackage.remove(track);
480 }
481 }
482 }
483 }
484 return queueTime;
485 }
486
487
488
489
490
491
492
493
494
495
496
497 private void createCopyOfTrack(MediaPackage mediaPackage, Track track, MediaPackageElementFlavor targetFlavor)
498 throws IllegalArgumentException, NotFoundException,IOException {
499
500 MediaPackageElementFlavor targetCopyFlavor = null;
501 if (track.hasVideo()) {
502 targetCopyFlavor = targetFlavor;
503 } else {
504 targetCopyFlavor = deriveAudioFlavor(targetFlavor);
505 }
506 logger.debug("Copying track {} with flavor {} using target flavor {}", track.getURI(), track.getFlavor(), targetCopyFlavor);
507 copyPartialToSource(mediaPackage, targetCopyFlavor, track);
508 }
509
510
511
512
513
514
515
516
517
518
519
520
521 private void adjustAudioTrackTargetFlavor(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor)
522 throws IllegalArgumentException, NotFoundException,IOException {
523
524 Track[] targetAudioTracks = mediaPackage.getTracks(deriveAudioFlavor(targetFlavor));
525 for (Track track : targetAudioTracks) {
526 logger.debug("Adding {} to finished audio tracks.", track.getURI());
527 mediaPackage.remove(track);
528 track.setFlavor(targetFlavor);
529 mediaPackage.add(track);
530 }
531 }
532
533 private TrackSelector mkTrackSelector(Optional<String> flavor) throws WorkflowOperationException {
534 final TrackSelector s = new TrackSelector();
535 if (flavor.isPresent()) {
536 try {
537 final MediaPackageElementFlavor f = MediaPackageElementFlavor.parseFlavor(flavor.get());
538 s.addFlavor(f);
539 s.addFlavor(deriveAudioFlavor(f));
540 } catch (IllegalArgumentException e) {
541 throw new WorkflowOperationException("Flavor '" + flavor.get() + "' is malformed");
542 }
543 }
544 return s;
545 }
546
547
548
549
550
551
552
553
554
555
556
557 protected Job startConcatJob(EncodingProfile profile, List<Track> tracks, float outputFramerate, boolean forceDivisible)
558 throws MediaPackageException, EncoderException {
559 final Dimension dim = determineDimension(tracks, forceDivisible);
560 if (outputFramerate > 0.0) {
561 return composerService.concat(profile.getIdentifier(), dim, outputFramerate, true, Collections.toArray(Track.class, tracks));
562 } else {
563 return composerService.concat(profile.getIdentifier(), dim, true, Collections.toArray(Track.class, tracks)); }
564 }
565
566
567
568
569
570
571
572 protected static boolean trackNeedsTobeEncodedToStandard(Track track, List<String> requiredExtensions) {
573 String extension = FilenameUtils.getExtension(track.getURI().toString());
574 for (String requiredExtension : requiredExtensions) {
575 if (requiredExtension.equalsIgnoreCase(extension)) {
576 return false;
577 }
578 }
579 return true;
580 }
581
582
583
584
585
586
587
588
589 protected List<String> getRequiredExtensions(WorkflowOperationInstance operation) {
590 List<String> requiredExtensions = new ArrayList<String>();
591 String configExtensions = null;
592 try {
593 configExtensions = StringUtils.trimToNull(getConfig(operation, REQUIRED_EXTENSIONS));
594 } catch (WorkflowOperationException e) {
595 logger.info(
596 "Required extensions configuration key not specified so will be using default '{}'. Any input file not matching this extension will be re-encoded.",
597 DEFAULT_REQUIRED_EXTENSION);
598 }
599 if (configExtensions != null) {
600 String[] extensions = configExtensions.split(",");
601 for (String extension : extensions) {
602 requiredExtensions.add(extension);
603 }
604 }
605 if (requiredExtensions.size() == 0) {
606 requiredExtensions.add(DEFAULT_REQUIRED_EXTENSION);
607 }
608 return requiredExtensions;
609 }
610
611
612
613
614
615
616
617
618 protected Optional<EncodingProfile> getForceEncodingProfile(WorkflowOperationInstance woi, boolean forceEncoding)
619 throws WorkflowOperationException {
620 if (!forceEncoding) {
621 return Optional.empty();
622 }
623
624 Optional<String> profileNameOpt = getOptConfig(woi, FORCE_ENCODING_PROFILE);
625 if (forceEncoding && profileNameOpt.isEmpty()) {
626 throw new WorkflowOperationException("Force encoding profile must be set!");
627 }
628
629 String profileName = profileNameOpt.get();
630 EncodingProfile profile = composerService.getProfile(profileName);
631 if (profile == null) {
632 throw new WorkflowOperationException("Force encoding profile '" + profileName + "' was not found");
633 }
634
635 return Optional.of(profile);
636 }
637
638
639
640
641
642 private MediaPackageElementFlavor parseTargetFlavor(String flavor, String flavorType)
643 throws WorkflowOperationException {
644 final MediaPackageElementFlavor targetFlavor;
645 try {
646 targetFlavor = MediaPackageElementFlavor.parseFlavor(flavor);
647 if ("*".equals(targetFlavor.getType()) || "*".equals(targetFlavor.getSubtype())) {
648 throw new WorkflowOperationException(format(
649 "Target %s flavor must have a type and a subtype, '*' are not allowed!", flavorType));
650 }
651 } catch (IllegalArgumentException e) {
652 throw new WorkflowOperationException(format("Target %s flavor '%s' is malformed", flavorType, flavor));
653 }
654 return targetFlavor;
655 }
656
657
658 private MediaPackageElementFlavor deriveAudioFlavor(MediaPackageElementFlavor flavor) {
659 return MediaPackageElementFlavor.flavor(flavor.getType().concat(FLAVOR_AUDIO_SUFFIX), flavor.getSubtype());
660 }
661
662
663
664
665
666
667
668
669
670
671 private Dimension determineDimension(List<Track> tracks, boolean forceDivisible) {
672 Tuple<Track, Dimension> trackDimension = getLargestTrack(tracks);
673 if (trackDimension == null)
674 return null;
675
676 if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
677 Dimension scaledDimension = Dimension.dimension((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
678 .getB().getHeight() / 2) * 2);
679 logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
680 trackDimension.getB(), trackDimension.getA());
681 return scaledDimension;
682 } else {
683 logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
684 return trackDimension.getB();
685 }
686 }
687
688
689
690
691
692
693
694
695 private Tuple<Track, Dimension> getLargestTrack(List<Track> tracks) {
696 Track track = null;
697 Dimension dimension = null;
698 for (Track t : tracks) {
699 if (!t.hasVideo())
700 continue;
701
702 VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
703 int frameWidth = videoStreams[0].getFrameWidth();
704 int frameHeight = videoStreams[0].getFrameHeight();
705 if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
706 dimension = Dimension.dimension(frameWidth, frameHeight);
707 track = t;
708 }
709 }
710 if (track == null || dimension == null)
711 return null;
712
713 return Tuple.tuple(track, dimension);
714 }
715
716 private long checkForTrimming(MediaPackage mediaPackage, EncodingProfile trimProfile,
717 MediaPackageElementFlavor targetFlavor, Float videoDuration, List<MediaPackageElement> elementsToClean)
718 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
719 ServiceRegistryException, IOException {
720 MediaPackageElement[] elements = mediaPackage.getElementsByFlavor(targetFlavor);
721 if (elements.length == 0)
722 return 0;
723
724 Track trackToTrim = (Track) elements[0];
725 if (elements.length == 1 && trackToTrim.getDuration() / 1000 > videoDuration) {
726 Long trimSeconds = (long) (trackToTrim.getDuration() / 1000 - videoDuration);
727 logger.info("Shorten track {} to target duration {} by {} seconds",
728 trackToTrim.toString(), videoDuration.toString(), trimSeconds.toString());
729 return trimEnd(mediaPackage, trimProfile, trackToTrim, videoDuration, elementsToClean);
730 } else if (elements.length > 1) {
731 logger.warn("Multiple tracks with flavor {} found! Trimming not possible!", targetFlavor);
732 }
733 return 0;
734 }
735
736 private List<Track> getPureVideoTracks(MediaPackage mediaPackage, MediaPackageElementFlavor videoFlavor) {
737 return Arrays.stream(mediaPackage.getTracks())
738 .filter(track -> track.getFlavor().matches(videoFlavor))
739 .filter(Track::hasVideo)
740 .filter(track -> !track.hasAudio())
741 .collect(Collectors.toList());
742 }
743
744 private List<Track> getPureAudioTracks(MediaPackage mediaPackage, MediaPackageElementFlavor audioFlavor) {
745 return Arrays.stream(mediaPackage.getTracks())
746 .filter(track -> track.getFlavor().matches(audioFlavor))
747 .filter(Track::hasAudio)
748 .filter(track -> !track.hasVideo())
749 .collect(Collectors.toList());
750 }
751
752 protected long checkForMuxing(MediaPackage mediaPackage, MediaPackageElementFlavor targetPresentationFlavor,
753 MediaPackageElementFlavor targetPresenterFlavor, boolean useSuffix, List<MediaPackageElement> elementsToClean)
754 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
755 ServiceRegistryException, IOException {
756
757 long queueTime = 0L;
758
759 List<Track> videoElements = getPureVideoTracks(mediaPackage, targetPresentationFlavor);
760 List<Track> audioElements;
761 if (useSuffix) {
762 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresentationFlavor));
763 } else {
764 audioElements = getPureAudioTracks(mediaPackage, targetPresentationFlavor);
765 }
766
767 Track videoTrack = null;
768 Track audioTrack = null;
769
770 if (videoElements.size() == 1 && audioElements.size() == 0) {
771 videoTrack = videoElements.get(0);
772 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
773 audioTrack = audioElements.get(0);
774 }
775
776 videoElements = getPureVideoTracks(mediaPackage, targetPresenterFlavor);
777 if (useSuffix) {
778 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresenterFlavor));
779 } else {
780 audioElements = getPureAudioTracks(mediaPackage, targetPresenterFlavor);
781 }
782
783 if (videoElements.size() == 1 && audioElements.size() == 0) {
784 videoTrack = videoElements.get(0);
785 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
786 audioTrack = audioElements.get(0);
787 }
788
789 logger.debug("Check for mux between '{}' and '{}' flavors and found video track '{}' and audio track '{}'",
790 targetPresentationFlavor, targetPresenterFlavor, videoTrack, audioTrack);
791 if (videoTrack != null && audioTrack != null) {
792 queueTime += mux(mediaPackage, videoTrack, audioTrack, elementsToClean);
793 return queueTime;
794 } else {
795 return queueTime;
796 }
797 }
798
799
800
801
802
803
804
805 protected long mux(MediaPackage mediaPackage, Track video, Track audio, List<MediaPackageElement> elementsToClean)
806 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
807 ServiceRegistryException, IOException {
808 logger.debug("Muxing video {} and audio {}", video.getURI(), audio.getURI());
809 Job muxJob = composerService.mux(video, audio, PrepareAVWorkflowOperationHandler.MUX_AV_PROFILE);
810 if (!waitForStatus(muxJob).isSuccess()) {
811 throw new WorkflowOperationException("Muxing of audio " + audio + " and video " + video + " failed");
812 }
813 muxJob = serviceRegistry.getJob(muxJob.getId());
814
815 final Track muxed = (Track) MediaPackageElementParser.getFromXml(muxJob.getPayload());
816 if (muxed == null) {
817 throw new WorkflowOperationException("Muxed job " + muxJob + " returned no payload!");
818 }
819 muxed.setFlavor(video.getFlavor());
820 muxed.setURI(workspace.moveTo(muxed.getURI(), mediaPackage.getIdentifier().toString(), muxed.getIdentifier(),
821 FilenameUtils.getName(video.getURI().toString())));
822 elementsToClean.add(audio);
823 mediaPackage.remove(audio);
824 elementsToClean.add(video);
825 mediaPackage.remove(video);
826 mediaPackage.add(muxed);
827 return muxJob.getQueueTime();
828 }
829
830 private void copyPartialToSource(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor, Track track)
831 throws NotFoundException, IOException {
832 FileInputStream in = null;
833 try {
834 Track copyTrack = (Track) track.clone();
835 File originalFile = workspace.get(copyTrack.getURI());
836 in = new FileInputStream(originalFile);
837
838 copyTrack.generateIdentifier();
839 copyTrack.setURI(workspace.put(mediaPackage.getIdentifier().toString(), copyTrack.getIdentifier(),
840 FilenameUtils.getName(copyTrack.getURI().toString()), in));
841 copyTrack.setFlavor(targetFlavor);
842 copyTrack.referTo(track);
843 mediaPackage.add(copyTrack);
844 logger.info("Copied partial source element {} to {} with target flavor {}", track.toString(),
845 copyTrack.toString(), targetFlavor.toString());
846 } finally {
847 IOUtils.closeQuietly(in);
848 }
849 }
850
851
852
853
854
855
856
857
858 private List<Track> preencode(EncodingProfile profile, List<Track> tracks)
859 throws MediaPackageException, EncoderException, WorkflowOperationException, NotFoundException,
860 ServiceRegistryException {
861 List<Track> encodedTracks = new ArrayList<>();
862 for (Track track : tracks) {
863 logger.info("Preencoding track {}", track.getIdentifier());
864 Job encodeJob = composerService.encode(track, profile.getIdentifier());
865 if (!waitForStatus(encodeJob).isSuccess()) {
866 throw new WorkflowOperationException("Encoding of track " + track + " failed");
867 }
868 encodeJob = serviceRegistry.getJob(encodeJob.getId());
869 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
870 if (encodedTrack == null) {
871 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
872 }
873 encodedTrack.setIdentifier(track.getIdentifier());
874 encodedTrack.setFlavor(track.getFlavor());
875 encodedTracks.add(encodedTrack);
876 }
877
878 return encodedTracks;
879 }
880
881
882
883
884
885
886
887 private long encodeToStandard(MediaPackage mp, EncodingProfile profile, MediaPackageElementFlavor targetFlavor,
888 Track track) throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
889 ServiceRegistryException, IOException {
890 Job encodeJob = composerService.encode(track, profile.getIdentifier());
891 if (!waitForStatus(encodeJob).isSuccess()) {
892 throw new WorkflowOperationException("Encoding of track " + track + " failed");
893 }
894 encodeJob = serviceRegistry.getJob(encodeJob.getId());
895 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
896 if (encodedTrack == null) {
897 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
898 }
899 URI uri;
900 if (FilenameUtils.getExtension(encodedTrack.getURI().toString()).equalsIgnoreCase(
901 FilenameUtils.getExtension(track.getURI().toString()))) {
902 uri = workspace.moveTo(encodedTrack.getURI(), mp.getIdentifier().toString(), encodedTrack.getIdentifier(),
903 FilenameUtils.getName(track.getURI().toString()));
904 } else {
905
906 uri = workspace.moveTo(
907 encodedTrack.getURI(),
908 mp.getIdentifier().toString(),
909 encodedTrack.getIdentifier(),
910 FilenameUtils.getBaseName(track.getURI().toString()) + "."
911 + FilenameUtils.getExtension(encodedTrack.getURI().toString()));
912 }
913 encodedTrack.setURI(uri);
914 encodedTrack.setFlavor(targetFlavor);
915 mp.add(encodedTrack);
916 return encodeJob.getQueueTime();
917 }
918
919 private long trimEnd(MediaPackage mediaPackage, EncodingProfile trimProfile, Track track, double duration,
920 List<MediaPackageElement> elementsToClean) throws EncoderException, MediaPackageException,
921 WorkflowOperationException, NotFoundException, ServiceRegistryException, IOException {
922 Job trimJob = composerService.trim(track, trimProfile.getIdentifier(), 0, (long) (duration * 1000));
923 if (!waitForStatus(trimJob).isSuccess())
924 throw new WorkflowOperationException("Trimming of track " + track + " failed");
925
926 trimJob = serviceRegistry.getJob(trimJob.getId());
927
928 Track trimmedTrack = (Track) MediaPackageElementParser.getFromXml(trimJob.getPayload());
929 if (trimmedTrack == null)
930 throw new WorkflowOperationException("Trimming track " + track + " failed to produce a track");
931
932 URI uri = workspace.moveTo(trimmedTrack.getURI(), mediaPackage.getIdentifier().toString(),
933 trimmedTrack.getIdentifier(), FilenameUtils.getName(track.getURI().toString()));
934 trimmedTrack.setURI(uri);
935 trimmedTrack.setFlavor(track.getFlavor());
936
937 elementsToClean.add(track);
938 mediaPackage.remove(track);
939 mediaPackage.add(trimmedTrack);
940
941 return trimJob.getQueueTime();
942 }
943
944 private long processChildren(long position, List<Track> tracks, NodeList children, List<Track> originalTracks,
945 VCell<String> type, String mediaType, List<MediaPackageElement> elementsToClean, Long operationId)
946 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException, IOException {
947 for (int j = 0; j < children.getLength(); j++) {
948 Node item = children.item(j);
949 if (item.hasChildNodes()) {
950 position = processChildren(position, tracks, item.getChildNodes(), originalTracks, type, mediaType,
951 elementsToClean, operationId);
952 } else {
953 SMILMediaElement e = (SMILMediaElement) item;
954 if (mediaType.equals(e.getNodeName())) {
955 Track track;
956 try {
957 track = getFromOriginal(e.getId(), originalTracks, type);
958 } catch (IllegalStateException exception) {
959 logger.debug("Skipping smil entry, reason: " + exception.getMessage());
960 continue;
961 }
962 double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
963 long beginInMs = Math.round(beginInSeconds * 1000d);
964
965 if (beginInMs > position) {
966 double positionInSeconds = position / 1000d;
967 if (position == 0) {
968 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
969 logger.info("Extending {} audio track start by {} seconds silent audio", type.get(), beginInSeconds);
970 tracks.add(getSilentAudio(beginInSeconds, elementsToClean, operationId));
971 } else {
972 logger.info("Extending {} track start image frame by {} seconds", type.get(), beginInSeconds);
973 Attachment tempFirstImageFrame = extractImage(track, 0, elementsToClean);
974 tracks.add(createVideoFromImage(tempFirstImageFrame, beginInSeconds, elementsToClean));
975 }
976 position += beginInMs;
977 } else {
978 double fillTime = (beginInMs - position) / 1000d;
979 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
980 logger.info("Fill {} audio track gap from {} to {} with silent audio", type.get(),
981 Double.toString(positionInSeconds), Double.toString(beginInSeconds));
982 tracks.add(getSilentAudio(fillTime, elementsToClean, operationId));
983 } else {
984 logger.info("Fill {} track gap from {} to {} with image frame",
985 type.get(), Double.toString(positionInSeconds), Double.toString(beginInSeconds));
986 Track previousTrack = tracks.get(tracks.size() - 1);
987 Attachment tempLastImageFrame = extractLastImageFrame(previousTrack, elementsToClean);
988 tracks.add(createVideoFromImage(tempLastImageFrame, fillTime, elementsToClean));
989 }
990 position = beginInMs;
991 }
992 }
993 tracks.add(track);
994 position += Math.round(e.getDur() * 1000f);
995 }
996 }
997 }
998 return position;
999 }
1000
1001 private Track getFromOriginal(String trackId, List<Track> originalTracks, VCell<String> type) {
1002 for (Track t : originalTracks) {
1003 if (t.getIdentifier().contains(trackId)) {
1004 logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
1005 if (EMPTY_VALUE.equals(type.get())) {
1006 String suffix = (t.hasAudio() && !t.hasVideo()) ? FLAVOR_AUDIO_SUFFIX : "";
1007 type.set(t.getFlavor().getType() + suffix);
1008 }
1009 originalTracks.remove(t);
1010 return t;
1011 }
1012 }
1013 throw new IllegalStateException("No track matching smil Track-id: " + trackId);
1014 }
1015
1016 private Track getSilentAudio(final double time, final List<MediaPackageElement> elementsToClean,
1017 final Long operationId) throws EncoderException, MediaPackageException, WorkflowOperationException,
1018 NotFoundException, IOException {
1019 final URI uri = workspace.putInCollection(COLLECTION_ID, operationId + "-silent", new ByteArrayInputStream(
1020 EMPTY_VALUE.getBytes()));
1021 final Attachment emptyAttachment = (Attachment) MediaPackageElementBuilderFactory.newInstance().newElementBuilder()
1022 .elementFromURI(uri, Type.Attachment, MediaPackageElementFlavor.parseFlavor("audio/silent"));
1023 elementsToClean.add(emptyAttachment);
1024
1025 final Job silentAudioJob = composerService.imageToVideo(emptyAttachment, SILENT_AUDIO_PROFILE, time);
1026 if (!waitForStatus(silentAudioJob).isSuccess())
1027 throw new WorkflowOperationException("Silent audio job did not complete successfully");
1028
1029
1030 try {
1031 Optional<String> payloadOpt = getPayload(serviceRegistry, silentAudioJob);
1032 if (payloadOpt.isPresent()) {
1033 final Track silentAudio = (Track) MediaPackageElementParser.getFromXml(payloadOpt.get());
1034 elementsToClean.add(silentAudio);
1035 return silentAudio;
1036 }
1037
1038 throw new WorkflowOperationException(format("Job %s has no payload or cannot be updated", silentAudioJob));
1039 } catch (ServiceRegistryException ex) {
1040 throw new WorkflowOperationException(ex);
1041 }
1042 }
1043
1044 private Track createVideoFromImage(Attachment image, double time, List<MediaPackageElement> elementsToClean)
1045 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1046 Job imageToVideoJob = composerService.imageToVideo(image, IMAGE_MOVIE_PROFILE, time);
1047 if (!waitForStatus(imageToVideoJob).isSuccess())
1048 throw new WorkflowOperationException("Image to video job did not complete successfully");
1049
1050
1051 try {
1052 imageToVideoJob = serviceRegistry.getJob(imageToVideoJob.getId());
1053 } catch (ServiceRegistryException e) {
1054 throw new WorkflowOperationException(e);
1055 }
1056 Track imageVideo = (Track) MediaPackageElementParser.getFromXml(imageToVideoJob.getPayload());
1057 elementsToClean.add(imageVideo);
1058 return imageVideo;
1059 }
1060
1061 private Attachment extractImage(Track presentationTrack, double time, List<MediaPackageElement> elementsToClean)
1062 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1063 Job extractImageJob = composerService.image(presentationTrack, PREVIEW_PROFILE, time);
1064 if (!waitForStatus(extractImageJob).isSuccess())
1065 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1066
1067
1068 try {
1069 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1070 } catch (ServiceRegistryException e) {
1071 throw new WorkflowOperationException(e);
1072 }
1073 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1074 .get(0);
1075 elementsToClean.add(composedImages);
1076 return composedImages;
1077 }
1078
1079 private Attachment extractLastImageFrame(Track presentationTrack, List<MediaPackageElement> elementsToClean)
1080 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1081
1082 Map<String, String> properties = new HashMap<String, String>();
1083
1084 Job extractImageJob = composerService.image(presentationTrack, IMAGE_FRAME_PROFILE, properties);
1085 if (!waitForStatus(extractImageJob).isSuccess())
1086 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1087
1088
1089 try {
1090 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1091 } catch (ServiceRegistryException e) {
1092 throw new WorkflowOperationException(e);
1093 }
1094 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1095 .get(0);
1096 elementsToClean.add(composedImages);
1097 return composedImages;
1098 }
1099 }