1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.workflow.handler.composer;
22
23 import static java.lang.String.format;
24 import static org.opencastproject.util.JobUtil.getPayload;
25
26 import org.opencastproject.composer.api.ComposerService;
27 import org.opencastproject.composer.api.EncoderException;
28 import org.opencastproject.composer.api.EncodingProfile;
29 import org.opencastproject.composer.layout.Dimension;
30 import org.opencastproject.job.api.Job;
31 import org.opencastproject.job.api.JobContext;
32 import org.opencastproject.mediapackage.Attachment;
33 import org.opencastproject.mediapackage.MediaPackage;
34 import org.opencastproject.mediapackage.MediaPackageElement;
35 import org.opencastproject.mediapackage.MediaPackageElement.Type;
36 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
37 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
38 import org.opencastproject.mediapackage.MediaPackageElementParser;
39 import org.opencastproject.mediapackage.MediaPackageException;
40 import org.opencastproject.mediapackage.Track;
41 import org.opencastproject.mediapackage.TrackSupport;
42 import org.opencastproject.mediapackage.VideoStream;
43 import org.opencastproject.mediapackage.selector.TrackSelector;
44 import org.opencastproject.serviceregistry.api.ServiceRegistry;
45 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
46 import org.opencastproject.smil.api.util.SmilUtil;
47 import org.opencastproject.util.JobUtil;
48 import org.opencastproject.util.NotFoundException;
49 import org.opencastproject.util.data.Collections;
50 import org.opencastproject.util.data.Tuple;
51 import org.opencastproject.util.data.VCell;
52 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
53 import org.opencastproject.workflow.api.WorkflowInstance;
54 import org.opencastproject.workflow.api.WorkflowOperationException;
55 import org.opencastproject.workflow.api.WorkflowOperationHandler;
56 import org.opencastproject.workflow.api.WorkflowOperationInstance;
57 import org.opencastproject.workflow.api.WorkflowOperationResult;
58 import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
59 import org.opencastproject.workspace.api.Workspace;
60
61 import org.apache.commons.io.FilenameUtils;
62 import org.apache.commons.io.IOUtils;
63 import org.apache.commons.lang3.BooleanUtils;
64 import org.apache.commons.lang3.StringUtils;
65 import org.apache.commons.lang3.math.NumberUtils;
66 import org.osgi.service.component.annotations.Component;
67 import org.osgi.service.component.annotations.Reference;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70 import org.w3c.dom.Node;
71 import org.w3c.dom.NodeList;
72 import org.w3c.dom.smil.SMILDocument;
73 import org.w3c.dom.smil.SMILElement;
74 import org.w3c.dom.smil.SMILMediaElement;
75 import org.w3c.dom.smil.SMILParElement;
76 import org.xml.sax.SAXException;
77
78 import java.io.ByteArrayInputStream;
79 import java.io.File;
80 import java.io.FileInputStream;
81 import java.io.IOException;
82 import java.net.URI;
83 import java.util.ArrayList;
84 import java.util.Arrays;
85 import java.util.HashMap;
86 import java.util.List;
87 import java.util.Map;
88 import java.util.Map.Entry;
89 import java.util.Optional;
90 import java.util.stream.Collectors;
91
92
93
94
95 @Component(
96 immediate = true,
97 service = WorkflowOperationHandler.class,
98 property = {
99 "service.description=Partial import Workflow Operation Handler",
100 "workflow.operation=partial-import"
101 }
102 )
103 public class PartialImportWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
104
105
106 private static final String SOURCE_PRESENTER_FLAVOR = "source-presenter-flavor";
107 private static final String SOURCE_PRESENTATION_FLAVOR = "source-presentation-flavor";
108 private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
109
110 private static final String TARGET_PRESENTER_FLAVOR = "target-presenter-flavor";
111 private static final String TARGET_PRESENTATION_FLAVOR = "target-presentation-flavor";
112
113 private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
114 private static final String CONCAT_OUTPUT_FRAMERATE = "concat-output-framerate";
115 private static final String TRIM_ENCODING_PROFILE = "trim-encoding-profile";
116 private static final String FORCE_ENCODING_PROFILE = "force-encoding-profile";
117 private static final String PREENCODE_ENCODING_PROFILE = "preencode-encoding-profile";
118
119 private static final String FORCE_ENCODING = "force-encoding";
120 private static final String REQUIRED_EXTENSIONS = "required-extensions";
121 private static final String ENFORCE_DIVISIBLE_BY_TWO = "enforce-divisible-by-two";
122
123
124 private static final Logger logger = LoggerFactory.getLogger(PartialImportWorkflowOperationHandler.class);
125
126
127 private static final String EMPTY_VALUE = "";
128 private static final String NODE_TYPE_AUDIO = "audio";
129 private static final String NODE_TYPE_VIDEO = "video";
130 private static final String FLAVOR_AUDIO_SUFFIX = "-audio";
131 private static final String COLLECTION_ID = "composer";
132 private static final String UNKNOWN_KEY = "unknown";
133 private static final String PRESENTER_KEY = "presenter";
134 private static final String PRESENTATION_KEY = "presentation";
135 private static final String DEFAULT_REQUIRED_EXTENSION = "mp4";
136
137
138 private static final String PREVIEW_PROFILE = "import.preview";
139 private static final String IMAGE_FRAME_PROFILE = "import.image-frame";
140 private static final String SILENT_AUDIO_PROFILE = "import.silent";
141 private static final String IMAGE_MOVIE_PROFILE = "image-movie.work";
142
143
144 private ComposerService composerService = null;
145
146
147 private Workspace workspace = null;
148
149
150
151
152
153
154
155 @Reference
156 public void setComposerService(ComposerService composerService) {
157 this.composerService = composerService;
158 }
159
160
161
162
163
164
165
166
167 @Reference
168 public void setWorkspace(Workspace workspace) {
169 this.workspace = workspace;
170 }
171
172 @Reference
173 @Override
174 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
175 super.setServiceRegistry(serviceRegistry);
176 }
177
178
179
180
181
182
183
184 @Override
185 public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
186 throws WorkflowOperationException {
187 logger.debug("Running partial import workflow operation on workflow {}", workflowInstance.getId());
188
189 List<MediaPackageElement> elementsToClean = new ArrayList<MediaPackageElement>();
190
191 try {
192 return concat(workflowInstance.getMediaPackage(), workflowInstance.getCurrentOperation(), elementsToClean);
193 } catch (Exception e) {
194 throw new WorkflowOperationException(e);
195 } finally {
196 for (MediaPackageElement elem : elementsToClean) {
197 try {
198 workspace.delete(elem.getURI());
199 } catch (Exception e) {
200 logger.warn("Unable to delete element {}", elem, e);
201 }
202 }
203 }
204 }
205
206 private WorkflowOperationResult concat(MediaPackage src, WorkflowOperationInstance operation,
207 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, NotFoundException,
208 MediaPackageException, WorkflowOperationException, ServiceRegistryException {
209 final MediaPackage mediaPackage = (MediaPackage) src.clone();
210 final Long operationId = operation.getId();
211
212
213 final Optional<String> presenterFlavor = getOptConfig(operation, SOURCE_PRESENTER_FLAVOR);
214 final Optional<String> presentationFlavor = getOptConfig(operation, SOURCE_PRESENTATION_FLAVOR);
215 final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(getConfig(operation,
216 SOURCE_SMIL_FLAVOR));
217 final String concatEncodingProfile = getConfig(operation, CONCAT_ENCODING_PROFILE);
218 final Optional<String> concatOutputFramerate = getOptConfig(operation, CONCAT_OUTPUT_FRAMERATE);
219 final String trimEncodingProfile = getConfig(operation, TRIM_ENCODING_PROFILE);
220 final MediaPackageElementFlavor targetPresenterFlavor = parseTargetFlavor(
221 getConfig(operation, TARGET_PRESENTER_FLAVOR), "presenter");
222 final MediaPackageElementFlavor targetPresentationFlavor = parseTargetFlavor(
223 getConfig(operation, TARGET_PRESENTATION_FLAVOR), "presentation");
224 final boolean forceEncoding = BooleanUtils.toBoolean(getOptConfig(operation, FORCE_ENCODING).orElse("false"));
225 final Optional<EncodingProfile> forceProfile = getForceEncodingProfile(operation, forceEncoding);
226 final boolean forceDivisible = BooleanUtils.toBoolean(getOptConfig(operation, ENFORCE_DIVISIBLE_BY_TWO)
227 .orElse("false"));
228 final List<String> requiredExtensions = getRequiredExtensions(operation);
229 final String preencodeEncodingProfile = getConfig(operation, PREENCODE_ENCODING_PROFILE);
230
231
232
233
234 if (presenterFlavor.isEmpty() && presentationFlavor.isEmpty()) {
235 logger.warn("No presenter and presentation flavor has been set.");
236 return createResult(mediaPackage, Action.SKIP);
237 }
238
239 final EncodingProfile preencodeProfile = composerService.getProfile(preencodeEncodingProfile);
240 if (preencodeProfile == null) {
241 throw new WorkflowOperationException("Preencode encoding profile '" + preencodeEncodingProfile
242 + "' was not found");
243 }
244
245 final EncodingProfile concatProfile = composerService.getProfile(concatEncodingProfile);
246 if (concatProfile == null) {
247 throw new WorkflowOperationException("Concat encoding profile '" + concatEncodingProfile + "' was not found");
248 }
249
250 float outputFramerate = -1.0f;
251 if (concatOutputFramerate.isPresent()) {
252 if (NumberUtils.isNumber(concatOutputFramerate.get())) {
253 logger.info("Using concat output framerate");
254 outputFramerate = NumberUtils.toFloat(concatOutputFramerate.get());
255 } else {
256 throw new WorkflowOperationException("Unable to parse concat output frame rate!");
257 }
258 }
259
260 final EncodingProfile trimProfile = composerService.getProfile(trimEncodingProfile);
261 if (trimProfile == null) {
262 throw new WorkflowOperationException("Trim encoding profile '" + trimEncodingProfile + "' was not found");
263 }
264
265
266
267 final TrackSelector presenterTrackSelector = mkTrackSelector(presenterFlavor);
268 final TrackSelector presentationTrackSelector = mkTrackSelector(presentationFlavor);
269 List<Track> originalTracks = new ArrayList<Track>();
270
271 for (Track t : presenterTrackSelector.select(mediaPackage, false)) {
272 logger.info("Found partial presenter track {}", t);
273 originalTracks.add(t);
274 }
275
276 for (Track t : presentationTrackSelector.select(mediaPackage, false)) {
277 logger.info("Found partial presentation track {}", t);
278 originalTracks.add(t);
279 }
280
281
282 logger.info("Starting preencoding");
283 originalTracks = preencode(preencodeProfile, originalTracks);
284
285
286
287 final Map<String, Job> jobs = new HashMap<String, Job>();
288
289 final SMILDocument smilDocument;
290 try {
291 smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
292 } catch (SAXException e) {
293 throw new WorkflowOperationException(e);
294 }
295 final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
296 final NodeList sequences = parallel.getTimeChildren();
297 final float trackDurationInSeconds = parallel.getDur();
298 final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
299 for (int i = 0; i < sequences.getLength(); i++) {
300 final SMILElement item = (SMILElement) sequences.item(i);
301
302 for (final String mediaType : new String[] { NODE_TYPE_AUDIO, NODE_TYPE_VIDEO }) {
303 final List<Track> tracks = new ArrayList<Track>();
304 final VCell<String> sourceType = VCell.cell(EMPTY_VALUE);
305
306 final long position = processChildren(0, tracks, item.getChildNodes(), originalTracks, sourceType, mediaType,
307 elementsToClean, operationId);
308
309 if (tracks.isEmpty()) {
310 logger.debug("The tracks list was empty.");
311 continue;
312 }
313 final Track lastTrack = tracks.get(tracks.size() - 1);
314
315 if (position < trackDurationInMs) {
316 final double extendingTime = (trackDurationInMs - position) / 1000d;
317 if (extendingTime > 0) {
318 if (!lastTrack.hasVideo()) {
319 logger.info("Extending {} audio track end by {} seconds with silent audio", sourceType.get(),
320 extendingTime);
321 tracks.add(getSilentAudio(extendingTime, elementsToClean, operationId));
322 } else {
323 logger.info("Extending {} track end with last image frame by {} seconds",
324 sourceType.get(), extendingTime);
325 Attachment tempLastImageFrame = extractLastImageFrame(lastTrack, elementsToClean);
326 tracks.add(createVideoFromImage(tempLastImageFrame, extendingTime, elementsToClean));
327 }
328 }
329 }
330
331 if (tracks.size() < 2) {
332 logger.debug("There were less than 2 tracks, copying track...");
333 if (sourceType.get().startsWith(PRESENTER_KEY)) {
334 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresenterFlavor);
335 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
336 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresentationFlavor);
337 } else {
338 logger.warn("Can't handle unkown source type '{}' for unprocessed track", sourceType.get());
339 }
340 continue;
341 }
342
343 for (final Track t : tracks) {
344 if (!t.hasVideo() && !t.hasAudio()) {
345 logger.error("No audio or video stream available in the track with flavor {}! {}", t.getFlavor(), t);
346 throw new WorkflowOperationException("No audio or video stream available in the track " + t.toString());
347 }
348 }
349
350 if (sourceType.get().startsWith(PRESENTER_KEY)) {
351 logger.info("Concatenating {} track", PRESENTER_KEY);
352 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
353 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
354 logger.info("Concatenating {} track", PRESENTATION_KEY);
355 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
356 } else {
357 logger.warn("Can't handle unknown source type '{}'!", sourceType.get());
358 }
359 }
360 }
361
362
363 if (jobs.size() > 0) {
364 if (!JobUtil.waitForJobs(serviceRegistry, jobs.values()).isSuccess()) {
365 throw new WorkflowOperationException("One of the concat jobs did not complete successfully");
366 }
367 } else {
368 logger.info("No concatenating needed for presenter and presentation tracks, took partial source elements");
369 }
370
371
372 long queueTime = 0L;
373 MediaPackageElementFlavor adjustedTargetPresenterFlavor = targetPresenterFlavor;
374 MediaPackageElementFlavor adjustedTargetPresentationFlavor = targetPresentationFlavor;
375 for (final Entry<String, Job> job : jobs.entrySet()) {
376 final Optional<Job> concatJob = JobUtil.update(serviceRegistry, job.getValue());
377 if (concatJob.isPresent()) {
378 final String concatPayload = concatJob.get().getPayload();
379 if (concatPayload != null) {
380 final Track concatTrack;
381 try {
382 concatTrack = (Track) MediaPackageElementParser.getFromXml(concatPayload);
383 } catch (MediaPackageException e) {
384 throw new WorkflowOperationException(e);
385 }
386
387 final String fileName;
388
389
390 if (job.getKey().startsWith(PRESENTER_KEY)) {
391 if (!concatTrack.hasVideo()) {
392 fileName = PRESENTER_KEY.concat(FLAVOR_AUDIO_SUFFIX);
393 adjustedTargetPresenterFlavor = deriveAudioFlavor(targetPresenterFlavor);
394 } else {
395 fileName = PRESENTER_KEY;
396 adjustedTargetPresenterFlavor = targetPresenterFlavor;
397 }
398 concatTrack.setFlavor(adjustedTargetPresenterFlavor);
399 } else if (job.getKey().startsWith(PRESENTATION_KEY)) {
400 if (!concatTrack.hasVideo()) {
401 fileName = PRESENTATION_KEY.concat(FLAVOR_AUDIO_SUFFIX);
402 adjustedTargetPresentationFlavor = deriveAudioFlavor(targetPresentationFlavor);
403 } else {
404 fileName = PRESENTATION_KEY;
405 adjustedTargetPresentationFlavor = targetPresentationFlavor;
406 }
407 concatTrack.setFlavor(adjustedTargetPresentationFlavor);
408 } else {
409 fileName = UNKNOWN_KEY;
410 }
411
412 concatTrack.setURI(workspace.moveTo(concatTrack.getURI(), mediaPackage.getIdentifier().toString(),
413 concatTrack.getIdentifier(),
414 fileName + "." + FilenameUtils.getExtension(concatTrack.getURI().toString())));
415
416 logger.info("Concatenated track {} got flavor '{}'", concatTrack, concatTrack.getFlavor());
417
418 mediaPackage.add(concatTrack);
419 queueTime += concatJob.get().getQueueTime();
420 } else {
421
422 logger.warn("Concat job {} does not contain a payload", concatJob);
423 }
424 } else {
425 logger.warn("Concat job {} could not be updated since it cannot be found", job.getValue());
426 }
427 }
428
429
430 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresentationFlavor, trackDurationInSeconds,
431 elementsToClean);
432 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresentationFlavor),
433 trackDurationInSeconds, elementsToClean);
434 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresenterFlavor, trackDurationInSeconds,
435 elementsToClean);
436 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresenterFlavor),
437 trackDurationInSeconds, elementsToClean);
438
439
440 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, deriveAudioFlavor(targetPresenterFlavor),
441 false, elementsToClean);
442 queueTime += checkForMuxing(mediaPackage, targetPresentationFlavor, deriveAudioFlavor(targetPresentationFlavor),
443 false, elementsToClean);
444
445 adjustAudioTrackTargetFlavor(mediaPackage, targetPresenterFlavor);
446 adjustAudioTrackTargetFlavor(mediaPackage, targetPresentationFlavor);
447
448
449 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, targetPresentationFlavor, false, elementsToClean);
450
451 queueTime += checkForEncodeToStandard(mediaPackage, forceEncoding, forceProfile, requiredExtensions,
452 targetPresenterFlavor, targetPresentationFlavor, elementsToClean);
453
454 final WorkflowOperationResult result = createResult(mediaPackage, Action.CONTINUE, queueTime);
455 logger.debug("Partial import operation completed");
456 return result;
457 }
458
459 protected long checkForEncodeToStandard(MediaPackage mediaPackage, boolean forceEncoding,
460 Optional<EncodingProfile> forceProfile, List<String> requiredExtensions,
461 MediaPackageElementFlavor targetPresenterFlavor, MediaPackageElementFlavor targetPresentationFlavor,
462 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, MediaPackageException,
463 NotFoundException, ServiceRegistryException, WorkflowOperationException {
464 long queueTime = 0;
465 if (forceProfile.isPresent()) {
466 Track[] targetPresenterTracks = mediaPackage.getTracks(targetPresenterFlavor);
467 for (Track track : targetPresenterTracks) {
468 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
469 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
470 targetPresenterFlavor, track.getURI(), forceProfile.get());
471 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresenterFlavor, track);
472 elementsToClean.add(track);
473 mediaPackage.remove(track);
474 }
475 }
476
477 if (!targetPresenterFlavor.toString().equalsIgnoreCase(targetPresentationFlavor.toString())) {
478 Track[] targetPresentationTracks = mediaPackage.getTracks(targetPresentationFlavor);
479 for (Track track : targetPresentationTracks) {
480 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
481 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
482 targetPresentationFlavor, track.getURI(), forceProfile.get());
483 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresentationFlavor, track);
484 elementsToClean.add(track);
485 mediaPackage.remove(track);
486 }
487 }
488 }
489 }
490 return queueTime;
491 }
492
493
494
495
496
497
498
499
500
501
502
503 private void createCopyOfTrack(MediaPackage mediaPackage, Track track, MediaPackageElementFlavor targetFlavor)
504 throws IllegalArgumentException, NotFoundException,IOException {
505
506 MediaPackageElementFlavor targetCopyFlavor = null;
507 if (track.hasVideo()) {
508 targetCopyFlavor = targetFlavor;
509 } else {
510 targetCopyFlavor = deriveAudioFlavor(targetFlavor);
511 }
512 logger.debug("Copying track {} with flavor {} using target flavor {}", track.getURI(), track.getFlavor(),
513 targetCopyFlavor);
514 copyPartialToSource(mediaPackage, targetCopyFlavor, track);
515 }
516
517
518
519
520
521
522
523
524
525
526
527
528 private void adjustAudioTrackTargetFlavor(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor)
529 throws IllegalArgumentException, NotFoundException,IOException {
530
531 Track[] targetAudioTracks = mediaPackage.getTracks(deriveAudioFlavor(targetFlavor));
532 for (Track track : targetAudioTracks) {
533 logger.debug("Adding {} to finished audio tracks.", track.getURI());
534 mediaPackage.remove(track);
535 track.setFlavor(targetFlavor);
536 mediaPackage.add(track);
537 }
538 }
539
540 private TrackSelector mkTrackSelector(Optional<String> flavor) throws WorkflowOperationException {
541 final TrackSelector s = new TrackSelector();
542 if (flavor.isPresent()) {
543 try {
544 final MediaPackageElementFlavor f = MediaPackageElementFlavor.parseFlavor(flavor.get());
545 s.addFlavor(f);
546 s.addFlavor(deriveAudioFlavor(f));
547 } catch (IllegalArgumentException e) {
548 throw new WorkflowOperationException("Flavor '" + flavor.get() + "' is malformed");
549 }
550 }
551 return s;
552 }
553
554
555
556
557
558
559
560
561
562
563
564 protected Job startConcatJob(EncodingProfile profile, List<Track> tracks, float outputFramerate,
565 boolean forceDivisible)
566 throws MediaPackageException, EncoderException {
567 final Dimension dim = determineDimension(tracks, forceDivisible);
568 if (outputFramerate > 0.0) {
569 return composerService.concat(profile.getIdentifier(), dim, outputFramerate, true,
570 Collections.toArray(Track.class, tracks));
571 } else {
572 return composerService.concat(profile.getIdentifier(), dim, true, Collections.toArray(Track.class, tracks));
573 }
574 }
575
576
577
578
579
580
581
582 protected static boolean trackNeedsTobeEncodedToStandard(Track track, List<String> requiredExtensions) {
583 String extension = FilenameUtils.getExtension(track.getURI().toString());
584 for (String requiredExtension : requiredExtensions) {
585 if (requiredExtension.equalsIgnoreCase(extension)) {
586 return false;
587 }
588 }
589 return true;
590 }
591
592
593
594
595
596
597
598
599 protected List<String> getRequiredExtensions(WorkflowOperationInstance operation) {
600 List<String> requiredExtensions = new ArrayList<String>();
601 String configExtensions = null;
602 try {
603 configExtensions = StringUtils.trimToNull(getConfig(operation, REQUIRED_EXTENSIONS));
604 } catch (WorkflowOperationException e) {
605 logger.info(
606 "Required extensions configuration key not specified so will be using default '{}'. Any input file not "
607 + "matching this extension will be re-encoded.",
608 DEFAULT_REQUIRED_EXTENSION);
609 }
610 if (configExtensions != null) {
611 String[] extensions = configExtensions.split(",");
612 for (String extension : extensions) {
613 requiredExtensions.add(extension);
614 }
615 }
616 if (requiredExtensions.size() == 0) {
617 requiredExtensions.add(DEFAULT_REQUIRED_EXTENSION);
618 }
619 return requiredExtensions;
620 }
621
622
623
624
625
626
627
628
629 protected Optional<EncodingProfile> getForceEncodingProfile(WorkflowOperationInstance woi, boolean forceEncoding)
630 throws WorkflowOperationException {
631 if (!forceEncoding) {
632 return Optional.empty();
633 }
634
635 Optional<String> profileNameOpt = getOptConfig(woi, FORCE_ENCODING_PROFILE);
636 if (forceEncoding && profileNameOpt.isEmpty()) {
637 throw new WorkflowOperationException("Force encoding profile must be set!");
638 }
639
640 String profileName = profileNameOpt.get();
641 EncodingProfile profile = composerService.getProfile(profileName);
642 if (profile == null) {
643 throw new WorkflowOperationException("Force encoding profile '" + profileName + "' was not found");
644 }
645
646 return Optional.of(profile);
647 }
648
649
650
651
652
653 private MediaPackageElementFlavor parseTargetFlavor(String flavor, String flavorType)
654 throws WorkflowOperationException {
655 final MediaPackageElementFlavor targetFlavor;
656 try {
657 targetFlavor = MediaPackageElementFlavor.parseFlavor(flavor);
658 if ("*".equals(targetFlavor.getType()) || "*".equals(targetFlavor.getSubtype())) {
659 throw new WorkflowOperationException(format(
660 "Target %s flavor must have a type and a subtype, '*' are not allowed!", flavorType));
661 }
662 } catch (IllegalArgumentException e) {
663 throw new WorkflowOperationException(format("Target %s flavor '%s' is malformed", flavorType, flavor));
664 }
665 return targetFlavor;
666 }
667
668
669 private MediaPackageElementFlavor deriveAudioFlavor(MediaPackageElementFlavor flavor) {
670 return MediaPackageElementFlavor.flavor(flavor.getType().concat(FLAVOR_AUDIO_SUFFIX), flavor.getSubtype());
671 }
672
673
674
675
676
677
678
679
680
681
682 private Dimension determineDimension(List<Track> tracks, boolean forceDivisible) {
683 Tuple<Track, Dimension> trackDimension = getLargestTrack(tracks);
684 if (trackDimension == null) {
685 return null;
686 }
687
688 if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
689 Dimension scaledDimension = Dimension.dimension((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
690 .getB().getHeight() / 2) * 2);
691 logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
692 trackDimension.getB(), trackDimension.getA());
693 return scaledDimension;
694 } else {
695 logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
696 return trackDimension.getB();
697 }
698 }
699
700
701
702
703
704
705
706
707 private Tuple<Track, Dimension> getLargestTrack(List<Track> tracks) {
708 Track track = null;
709 Dimension dimension = null;
710 for (Track t : tracks) {
711 if (!t.hasVideo()) {
712 continue;
713 }
714
715 VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
716 int frameWidth = videoStreams[0].getFrameWidth();
717 int frameHeight = videoStreams[0].getFrameHeight();
718 if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
719 dimension = Dimension.dimension(frameWidth, frameHeight);
720 track = t;
721 }
722 }
723 if (track == null || dimension == null) {
724 return null;
725 }
726
727 return Tuple.tuple(track, dimension);
728 }
729
730 private long checkForTrimming(MediaPackage mediaPackage, EncodingProfile trimProfile,
731 MediaPackageElementFlavor targetFlavor, Float videoDuration, List<MediaPackageElement> elementsToClean)
732 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
733 ServiceRegistryException, IOException {
734 MediaPackageElement[] elements = mediaPackage.getElementsByFlavor(targetFlavor);
735 if (elements.length == 0) {
736 return 0;
737 }
738
739 Track trackToTrim = (Track) elements[0];
740 if (elements.length == 1 && trackToTrim.getDuration() / 1000 > videoDuration) {
741 Long trimSeconds = (long) (trackToTrim.getDuration() / 1000 - videoDuration);
742 logger.info("Shorten track {} to target duration {} by {} seconds",
743 trackToTrim.toString(), videoDuration.toString(), trimSeconds.toString());
744 return trimEnd(mediaPackage, trimProfile, trackToTrim, videoDuration, elementsToClean);
745 } else if (elements.length > 1) {
746 logger.warn("Multiple tracks with flavor {} found! Trimming not possible!", targetFlavor);
747 }
748 return 0;
749 }
750
751 private List<Track> getPureVideoTracks(MediaPackage mediaPackage, MediaPackageElementFlavor videoFlavor) {
752 return Arrays.stream(mediaPackage.getTracks())
753 .filter(track -> track.getFlavor().matches(videoFlavor))
754 .filter(Track::hasVideo)
755 .filter(track -> !track.hasAudio())
756 .collect(Collectors.toList());
757 }
758
759 private List<Track> getPureAudioTracks(MediaPackage mediaPackage, MediaPackageElementFlavor audioFlavor) {
760 return Arrays.stream(mediaPackage.getTracks())
761 .filter(track -> track.getFlavor().matches(audioFlavor))
762 .filter(Track::hasAudio)
763 .filter(track -> !track.hasVideo())
764 .collect(Collectors.toList());
765 }
766
767 protected long checkForMuxing(MediaPackage mediaPackage, MediaPackageElementFlavor targetPresentationFlavor,
768 MediaPackageElementFlavor targetPresenterFlavor, boolean useSuffix, List<MediaPackageElement> elementsToClean)
769 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
770 ServiceRegistryException, IOException {
771
772 long queueTime = 0L;
773
774 List<Track> videoElements = getPureVideoTracks(mediaPackage, targetPresentationFlavor);
775 List<Track> audioElements;
776 if (useSuffix) {
777 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresentationFlavor));
778 } else {
779 audioElements = getPureAudioTracks(mediaPackage, targetPresentationFlavor);
780 }
781
782 Track videoTrack = null;
783 Track audioTrack = null;
784
785 if (videoElements.size() == 1 && audioElements.size() == 0) {
786 videoTrack = videoElements.get(0);
787 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
788 audioTrack = audioElements.get(0);
789 }
790
791 videoElements = getPureVideoTracks(mediaPackage, targetPresenterFlavor);
792 if (useSuffix) {
793 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresenterFlavor));
794 } else {
795 audioElements = getPureAudioTracks(mediaPackage, targetPresenterFlavor);
796 }
797
798 if (videoElements.size() == 1 && audioElements.size() == 0) {
799 videoTrack = videoElements.get(0);
800 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
801 audioTrack = audioElements.get(0);
802 }
803
804 logger.debug("Check for mux between '{}' and '{}' flavors and found video track '{}' and audio track '{}'",
805 targetPresentationFlavor, targetPresenterFlavor, videoTrack, audioTrack);
806 if (videoTrack != null && audioTrack != null) {
807 queueTime += mux(mediaPackage, videoTrack, audioTrack, elementsToClean);
808 return queueTime;
809 } else {
810 return queueTime;
811 }
812 }
813
814
815
816
817
818
819
820 protected long mux(MediaPackage mediaPackage, Track video, Track audio, List<MediaPackageElement> elementsToClean)
821 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
822 ServiceRegistryException, IOException {
823 logger.debug("Muxing video {} and audio {}", video.getURI(), audio.getURI());
824 Job muxJob = composerService.mux(video, audio, PrepareAVWorkflowOperationHandler.MUX_AV_PROFILE);
825 if (!waitForStatus(muxJob).isSuccess()) {
826 throw new WorkflowOperationException("Muxing of audio " + audio + " and video " + video + " failed");
827 }
828 muxJob = serviceRegistry.getJob(muxJob.getId());
829
830 final Track muxed = (Track) MediaPackageElementParser.getFromXml(muxJob.getPayload());
831 if (muxed == null) {
832 throw new WorkflowOperationException("Muxed job " + muxJob + " returned no payload!");
833 }
834 muxed.setFlavor(video.getFlavor());
835 muxed.setURI(workspace.moveTo(muxed.getURI(), mediaPackage.getIdentifier().toString(), muxed.getIdentifier(),
836 FilenameUtils.getName(video.getURI().toString())));
837 elementsToClean.add(audio);
838 mediaPackage.remove(audio);
839 elementsToClean.add(video);
840 mediaPackage.remove(video);
841 mediaPackage.add(muxed);
842 return muxJob.getQueueTime();
843 }
844
845 private void copyPartialToSource(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor, Track track)
846 throws NotFoundException, IOException {
847 FileInputStream in = null;
848 try {
849 Track copyTrack = (Track) track.clone();
850 File originalFile = workspace.get(copyTrack.getURI());
851 in = new FileInputStream(originalFile);
852
853 copyTrack.generateIdentifier();
854 copyTrack.setURI(workspace.put(mediaPackage.getIdentifier().toString(), copyTrack.getIdentifier(),
855 FilenameUtils.getName(copyTrack.getURI().toString()), in));
856 copyTrack.setFlavor(targetFlavor);
857 copyTrack.referTo(track);
858 mediaPackage.add(copyTrack);
859 logger.info("Copied partial source element {} to {} with target flavor {}", track.toString(),
860 copyTrack.toString(), targetFlavor.toString());
861 } finally {
862 IOUtils.closeQuietly(in);
863 }
864 }
865
866
867
868
869
870
871
872
873 private List<Track> preencode(EncodingProfile profile, List<Track> tracks)
874 throws MediaPackageException, EncoderException, WorkflowOperationException, NotFoundException,
875 ServiceRegistryException {
876 List<Track> encodedTracks = new ArrayList<>();
877 for (Track track : tracks) {
878 logger.info("Preencoding track {}", track.getIdentifier());
879 Job encodeJob = composerService.encode(track, profile.getIdentifier());
880 if (!waitForStatus(encodeJob).isSuccess()) {
881 throw new WorkflowOperationException("Encoding of track " + track + " failed");
882 }
883 encodeJob = serviceRegistry.getJob(encodeJob.getId());
884 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
885 if (encodedTrack == null) {
886 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
887 }
888 encodedTrack.setIdentifier(track.getIdentifier());
889 encodedTrack.setFlavor(track.getFlavor());
890 encodedTracks.add(encodedTrack);
891 }
892
893 return encodedTracks;
894 }
895
896
897
898
899
900
901
902 private long encodeToStandard(MediaPackage mp, EncodingProfile profile, MediaPackageElementFlavor targetFlavor,
903 Track track) throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
904 ServiceRegistryException, IOException {
905 Job encodeJob = composerService.encode(track, profile.getIdentifier());
906 if (!waitForStatus(encodeJob).isSuccess()) {
907 throw new WorkflowOperationException("Encoding of track " + track + " failed");
908 }
909 encodeJob = serviceRegistry.getJob(encodeJob.getId());
910 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
911 if (encodedTrack == null) {
912 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
913 }
914 URI uri;
915 if (FilenameUtils.getExtension(encodedTrack.getURI().toString()).equalsIgnoreCase(
916 FilenameUtils.getExtension(track.getURI().toString()))) {
917 uri = workspace.moveTo(encodedTrack.getURI(), mp.getIdentifier().toString(), encodedTrack.getIdentifier(),
918 FilenameUtils.getName(track.getURI().toString()));
919 } else {
920
921 uri = workspace.moveTo(
922 encodedTrack.getURI(),
923 mp.getIdentifier().toString(),
924 encodedTrack.getIdentifier(),
925 FilenameUtils.getBaseName(track.getURI().toString()) + "."
926 + FilenameUtils.getExtension(encodedTrack.getURI().toString()));
927 }
928 encodedTrack.setURI(uri);
929 encodedTrack.setFlavor(targetFlavor);
930 mp.add(encodedTrack);
931 return encodeJob.getQueueTime();
932 }
933
934 private long trimEnd(MediaPackage mediaPackage, EncodingProfile trimProfile, Track track, double duration,
935 List<MediaPackageElement> elementsToClean) throws EncoderException, MediaPackageException,
936 WorkflowOperationException, NotFoundException, ServiceRegistryException, IOException {
937 Job trimJob = composerService.trim(track, trimProfile.getIdentifier(), 0, (long) (duration * 1000));
938 if (!waitForStatus(trimJob).isSuccess()) {
939 throw new WorkflowOperationException("Trimming of track " + track + " failed");
940 }
941
942 trimJob = serviceRegistry.getJob(trimJob.getId());
943
944 Track trimmedTrack = (Track) MediaPackageElementParser.getFromXml(trimJob.getPayload());
945 if (trimmedTrack == null) {
946 throw new WorkflowOperationException("Trimming track " + track + " failed to produce a track");
947 }
948
949 URI uri = workspace.moveTo(trimmedTrack.getURI(), mediaPackage.getIdentifier().toString(),
950 trimmedTrack.getIdentifier(), FilenameUtils.getName(track.getURI().toString()));
951 trimmedTrack.setURI(uri);
952 trimmedTrack.setFlavor(track.getFlavor());
953
954 elementsToClean.add(track);
955 mediaPackage.remove(track);
956 mediaPackage.add(trimmedTrack);
957
958 return trimJob.getQueueTime();
959 }
960
961 private long processChildren(long position, List<Track> tracks, NodeList children, List<Track> originalTracks,
962 VCell<String> type, String mediaType, List<MediaPackageElement> elementsToClean, Long operationId)
963 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException, IOException {
964 for (int j = 0; j < children.getLength(); j++) {
965 Node item = children.item(j);
966 if (item.hasChildNodes()) {
967 position = processChildren(position, tracks, item.getChildNodes(), originalTracks, type, mediaType,
968 elementsToClean, operationId);
969 } else {
970 SMILMediaElement e = (SMILMediaElement) item;
971 if (mediaType.equals(e.getNodeName())) {
972 Track track;
973 try {
974 track = getFromOriginal(e.getId(), originalTracks, type);
975 } catch (IllegalStateException exception) {
976 logger.debug("Skipping smil entry, reason: " + exception.getMessage());
977 continue;
978 }
979 double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
980 long beginInMs = Math.round(beginInSeconds * 1000d);
981
982 if (beginInMs > position) {
983 double positionInSeconds = position / 1000d;
984 if (position == 0) {
985 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
986 logger.info("Extending {} audio track start by {} seconds silent audio", type.get(), beginInSeconds);
987 tracks.add(getSilentAudio(beginInSeconds, elementsToClean, operationId));
988 } else {
989 logger.info("Extending {} track start image frame by {} seconds", type.get(), beginInSeconds);
990 Attachment tempFirstImageFrame = extractImage(track, 0, elementsToClean);
991 tracks.add(createVideoFromImage(tempFirstImageFrame, beginInSeconds, elementsToClean));
992 }
993 position += beginInMs;
994 } else {
995 double fillTime = (beginInMs - position) / 1000d;
996 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
997 logger.info("Fill {} audio track gap from {} to {} with silent audio", type.get(),
998 Double.toString(positionInSeconds), Double.toString(beginInSeconds));
999 tracks.add(getSilentAudio(fillTime, elementsToClean, operationId));
1000 } else {
1001 logger.info("Fill {} track gap from {} to {} with image frame",
1002 type.get(), Double.toString(positionInSeconds), Double.toString(beginInSeconds));
1003 Track previousTrack = tracks.get(tracks.size() - 1);
1004 Attachment tempLastImageFrame = extractLastImageFrame(previousTrack, elementsToClean);
1005 tracks.add(createVideoFromImage(tempLastImageFrame, fillTime, elementsToClean));
1006 }
1007 position = beginInMs;
1008 }
1009 }
1010 tracks.add(track);
1011 position += Math.round(e.getDur() * 1000f);
1012 }
1013 }
1014 }
1015 return position;
1016 }
1017
1018 private Track getFromOriginal(String trackId, List<Track> originalTracks, VCell<String> type) {
1019 for (Track t : originalTracks) {
1020 if (t.getIdentifier().contains(trackId)) {
1021 logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
1022 if (EMPTY_VALUE.equals(type.get())) {
1023 String suffix = (t.hasAudio() && !t.hasVideo()) ? FLAVOR_AUDIO_SUFFIX : "";
1024 type.set(t.getFlavor().getType() + suffix);
1025 }
1026 originalTracks.remove(t);
1027 return t;
1028 }
1029 }
1030 throw new IllegalStateException("No track matching smil Track-id: " + trackId);
1031 }
1032
1033 private Track getSilentAudio(final double time, final List<MediaPackageElement> elementsToClean,
1034 final Long operationId) throws EncoderException, MediaPackageException, WorkflowOperationException,
1035 NotFoundException, IOException {
1036 final URI uri = workspace.putInCollection(COLLECTION_ID, operationId + "-silent", new ByteArrayInputStream(
1037 EMPTY_VALUE.getBytes()));
1038 final Attachment emptyAttachment = (Attachment) MediaPackageElementBuilderFactory.newInstance().newElementBuilder()
1039 .elementFromURI(uri, Type.Attachment, MediaPackageElementFlavor.parseFlavor("audio/silent"));
1040 elementsToClean.add(emptyAttachment);
1041
1042 final Job silentAudioJob = composerService.imageToVideo(emptyAttachment, SILENT_AUDIO_PROFILE, time);
1043 if (!waitForStatus(silentAudioJob).isSuccess()) {
1044 throw new WorkflowOperationException("Silent audio job did not complete successfully");
1045 }
1046
1047
1048 try {
1049 Optional<String> payloadOpt = getPayload(serviceRegistry, silentAudioJob);
1050 if (payloadOpt.isPresent()) {
1051 final Track silentAudio = (Track) MediaPackageElementParser.getFromXml(payloadOpt.get());
1052 elementsToClean.add(silentAudio);
1053 return silentAudio;
1054 }
1055
1056 throw new WorkflowOperationException(format("Job %s has no payload or cannot be updated", silentAudioJob));
1057 } catch (ServiceRegistryException ex) {
1058 throw new WorkflowOperationException(ex);
1059 }
1060 }
1061
1062 private Track createVideoFromImage(Attachment image, double time, List<MediaPackageElement> elementsToClean)
1063 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1064 Job imageToVideoJob = composerService.imageToVideo(image, IMAGE_MOVIE_PROFILE, time);
1065 if (!waitForStatus(imageToVideoJob).isSuccess()) {
1066 throw new WorkflowOperationException("Image to video job did not complete successfully");
1067 }
1068
1069
1070 try {
1071 imageToVideoJob = serviceRegistry.getJob(imageToVideoJob.getId());
1072 } catch (ServiceRegistryException e) {
1073 throw new WorkflowOperationException(e);
1074 }
1075 Track imageVideo = (Track) MediaPackageElementParser.getFromXml(imageToVideoJob.getPayload());
1076 elementsToClean.add(imageVideo);
1077 return imageVideo;
1078 }
1079
1080 private Attachment extractImage(Track presentationTrack, double time, List<MediaPackageElement> elementsToClean)
1081 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1082 Job extractImageJob = composerService.image(presentationTrack, PREVIEW_PROFILE, time);
1083 if (!waitForStatus(extractImageJob).isSuccess()) {
1084 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1085 }
1086
1087
1088 try {
1089 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1090 } catch (ServiceRegistryException e) {
1091 throw new WorkflowOperationException(e);
1092 }
1093 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1094 .get(0);
1095 elementsToClean.add(composedImages);
1096 return composedImages;
1097 }
1098
1099 private Attachment extractLastImageFrame(Track presentationTrack, List<MediaPackageElement> elementsToClean)
1100 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1101
1102 Map<String, String> properties = new HashMap<String, String>();
1103
1104 Job extractImageJob = composerService.image(presentationTrack, IMAGE_FRAME_PROFILE, properties);
1105 if (!waitForStatus(extractImageJob).isSuccess()) {
1106 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1107 }
1108
1109
1110 try {
1111 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1112 } catch (ServiceRegistryException e) {
1113 throw new WorkflowOperationException(e);
1114 }
1115 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1116 .get(0);
1117 elementsToClean.add(composedImages);
1118 return composedImages;
1119 }
1120 }