1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.workflow.handler.composer;
22
23 import static java.lang.String.format;
24 import static org.opencastproject.util.JobUtil.getPayload;
25
26 import org.opencastproject.composer.api.ComposerService;
27 import org.opencastproject.composer.api.EncoderException;
28 import org.opencastproject.composer.api.EncodingProfile;
29 import org.opencastproject.composer.layout.Dimension;
30 import org.opencastproject.job.api.Job;
31 import org.opencastproject.job.api.JobContext;
32 import org.opencastproject.mediapackage.Attachment;
33 import org.opencastproject.mediapackage.MediaPackage;
34 import org.opencastproject.mediapackage.MediaPackageElement;
35 import org.opencastproject.mediapackage.MediaPackageElement.Type;
36 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
37 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
38 import org.opencastproject.mediapackage.MediaPackageElementParser;
39 import org.opencastproject.mediapackage.MediaPackageException;
40 import org.opencastproject.mediapackage.Track;
41 import org.opencastproject.mediapackage.TrackSupport;
42 import org.opencastproject.mediapackage.VideoStream;
43 import org.opencastproject.mediapackage.selector.TrackSelector;
44 import org.opencastproject.serviceregistry.api.ServiceRegistry;
45 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
46 import org.opencastproject.smil.api.util.SmilUtil;
47 import org.opencastproject.util.JobUtil;
48 import org.opencastproject.util.NotFoundException;
49 import org.opencastproject.util.data.Collections;
50 import org.opencastproject.util.data.Tuple;
51 import org.opencastproject.util.data.VCell;
52 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
53 import org.opencastproject.workflow.api.WorkflowInstance;
54 import org.opencastproject.workflow.api.WorkflowOperationException;
55 import org.opencastproject.workflow.api.WorkflowOperationHandler;
56 import org.opencastproject.workflow.api.WorkflowOperationInstance;
57 import org.opencastproject.workflow.api.WorkflowOperationResult;
58 import org.opencastproject.workflow.api.WorkflowOperationResult.Action;
59 import org.opencastproject.workspace.api.Workspace;
60
61 import org.apache.commons.io.FilenameUtils;
62 import org.apache.commons.io.IOUtils;
63 import org.apache.commons.lang3.BooleanUtils;
64 import org.apache.commons.lang3.StringUtils;
65 import org.apache.commons.lang3.math.NumberUtils;
66 import org.osgi.service.component.annotations.Component;
67 import org.osgi.service.component.annotations.Reference;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70 import org.w3c.dom.Node;
71 import org.w3c.dom.NodeList;
72 import org.w3c.dom.smil.SMILDocument;
73 import org.w3c.dom.smil.SMILElement;
74 import org.w3c.dom.smil.SMILMediaElement;
75 import org.w3c.dom.smil.SMILParElement;
76 import org.xml.sax.SAXException;
77
78 import java.io.ByteArrayInputStream;
79 import java.io.File;
80 import java.io.FileInputStream;
81 import java.io.IOException;
82 import java.net.URI;
83 import java.util.ArrayList;
84 import java.util.Arrays;
85 import java.util.HashMap;
86 import java.util.List;
87 import java.util.Map;
88 import java.util.Map.Entry;
89 import java.util.Optional;
90 import java.util.UUID;
91 import java.util.stream.Collectors;
92
93
94
95
96 @Component(
97 immediate = true,
98 service = WorkflowOperationHandler.class,
99 property = {
100 "service.description=Partial import Workflow Operation Handler",
101 "workflow.operation=partial-import"
102 }
103 )
104 public class PartialImportWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
105
106
107 private static final String SOURCE_PRESENTER_FLAVOR = "source-presenter-flavor";
108 private static final String SOURCE_PRESENTATION_FLAVOR = "source-presentation-flavor";
109 private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
110
111 private static final String TARGET_PRESENTER_FLAVOR = "target-presenter-flavor";
112 private static final String TARGET_PRESENTATION_FLAVOR = "target-presentation-flavor";
113
114 private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
115 private static final String CONCAT_OUTPUT_FRAMERATE = "concat-output-framerate";
116 private static final String TRIM_ENCODING_PROFILE = "trim-encoding-profile";
117 private static final String FORCE_ENCODING_PROFILE = "force-encoding-profile";
118 private static final String PREENCODE_ENCODING_PROFILE = "preencode-encoding-profile";
119
120 private static final String FORCE_ENCODING = "force-encoding";
121 private static final String REQUIRED_EXTENSIONS = "required-extensions";
122 private static final String ENFORCE_DIVISIBLE_BY_TWO = "enforce-divisible-by-two";
123
124
125 private static final Logger logger = LoggerFactory.getLogger(PartialImportWorkflowOperationHandler.class);
126
127
128 private static final String EMPTY_VALUE = "";
129 private static final String NODE_TYPE_AUDIO = "audio";
130 private static final String NODE_TYPE_VIDEO = "video";
131 private static final String FLAVOR_AUDIO_SUFFIX = "-audio";
132 private static final String COLLECTION_ID = "composer";
133 private static final String UNKNOWN_KEY = "unknown";
134 private static final String PRESENTER_KEY = "presenter";
135 private static final String PRESENTATION_KEY = "presentation";
136 private static final String DEFAULT_REQUIRED_EXTENSION = "mp4";
137
138
139 private static final String PREVIEW_PROFILE = "import.preview";
140 private static final String IMAGE_FRAME_PROFILE = "import.image-frame";
141 private static final String SILENT_AUDIO_PROFILE = "import.silent";
142 private static final String IMAGE_MOVIE_PROFILE = "image-movie.work";
143
144
145 private ComposerService composerService = null;
146
147
148 private Workspace workspace = null;
149
150
151
152
153
154
155
156 @Reference
157 public void setComposerService(ComposerService composerService) {
158 this.composerService = composerService;
159 }
160
161
162
163
164
165
166
167
168 @Reference
169 public void setWorkspace(Workspace workspace) {
170 this.workspace = workspace;
171 }
172
173 @Reference
174 @Override
175 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
176 super.setServiceRegistry(serviceRegistry);
177 }
178
179
180
181
182
183
184
185 @Override
186 public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
187 throws WorkflowOperationException {
188 logger.debug("Running partial import workflow operation on workflow {}", workflowInstance.getId());
189
190 List<MediaPackageElement> elementsToClean = new ArrayList<MediaPackageElement>();
191
192 try {
193 return concat(workflowInstance.getMediaPackage(), workflowInstance.getCurrentOperation(), elementsToClean);
194 } catch (Exception e) {
195 throw new WorkflowOperationException(e);
196 } finally {
197 for (MediaPackageElement elem : elementsToClean) {
198 try {
199 workspace.delete(elem.getURI());
200 } catch (Exception e) {
201 logger.warn("Unable to delete element {}", elem, e);
202 }
203 }
204 }
205 }
206
207 private WorkflowOperationResult concat(MediaPackage src, WorkflowOperationInstance operation,
208 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, NotFoundException,
209 MediaPackageException, WorkflowOperationException, ServiceRegistryException {
210 final MediaPackage mediaPackage = (MediaPackage) src.clone();
211 final Long operationId = operation.getId();
212
213
214 final Optional<String> presenterFlavor = Optional.ofNullable(
215 getOptConfig(operation, SOURCE_PRESENTER_FLAVOR).orNull());
216 final Optional<String> presentationFlavor = Optional.ofNullable(
217 getOptConfig(operation, SOURCE_PRESENTATION_FLAVOR).orNull());
218 final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(getConfig(operation, SOURCE_SMIL_FLAVOR));
219 final String concatEncodingProfile = getConfig(operation, CONCAT_ENCODING_PROFILE);
220 final Optional<String> concatOutputFramerate = Optional.ofNullable(
221 getOptConfig(operation, CONCAT_OUTPUT_FRAMERATE).orNull());
222 final String trimEncodingProfile = getConfig(operation, TRIM_ENCODING_PROFILE);
223 final MediaPackageElementFlavor targetPresenterFlavor = parseTargetFlavor(
224 getConfig(operation, TARGET_PRESENTER_FLAVOR), "presenter");
225 final MediaPackageElementFlavor targetPresentationFlavor = parseTargetFlavor(
226 getConfig(operation, TARGET_PRESENTATION_FLAVOR), "presentation");
227 final boolean forceEncoding = BooleanUtils.toBoolean(getOptConfig(operation, FORCE_ENCODING).getOr("false"));
228 final Optional<EncodingProfile> forceProfile = getForceEncodingProfile(operation, forceEncoding);
229 final boolean forceDivisible = BooleanUtils.toBoolean(getOptConfig(operation, ENFORCE_DIVISIBLE_BY_TWO).getOr("false"));
230 final List<String> requiredExtensions = getRequiredExtensions(operation);
231 final String preencodeEncodingProfile = getConfig(operation, PREENCODE_ENCODING_PROFILE);
232
233
234
235
236 if (presenterFlavor.isEmpty() && presentationFlavor.isEmpty()) {
237 logger.warn("No presenter and presentation flavor has been set.");
238 return createResult(mediaPackage, Action.SKIP);
239 }
240
241 final EncodingProfile preencodeProfile = composerService.getProfile(preencodeEncodingProfile);
242 if (preencodeProfile == null) {
243 throw new WorkflowOperationException("Preencode encoding profile '" + preencodeEncodingProfile + "' was not found");
244 }
245
246 final EncodingProfile concatProfile = composerService.getProfile(concatEncodingProfile);
247 if (concatProfile == null) {
248 throw new WorkflowOperationException("Concat encoding profile '" + concatEncodingProfile + "' was not found");
249 }
250
251 float outputFramerate = -1.0f;
252 if (concatOutputFramerate.isPresent()) {
253 if (NumberUtils.isNumber(concatOutputFramerate.get())) {
254 logger.info("Using concat output framerate");
255 outputFramerate = NumberUtils.toFloat(concatOutputFramerate.get());
256 } else {
257 throw new WorkflowOperationException("Unable to parse concat output frame rate!");
258 }
259 }
260
261 final EncodingProfile trimProfile = composerService.getProfile(trimEncodingProfile);
262 if (trimProfile == null) {
263 throw new WorkflowOperationException("Trim encoding profile '" + trimEncodingProfile + "' was not found");
264 }
265
266
267
268 final TrackSelector presenterTrackSelector = mkTrackSelector(presenterFlavor);
269 final TrackSelector presentationTrackSelector = mkTrackSelector(presentationFlavor);
270 List<Track> originalTracks = new ArrayList<Track>();
271
272 for (Track t : presenterTrackSelector.select(mediaPackage, false)) {
273 logger.info("Found partial presenter track {}", t);
274 originalTracks.add(t);
275 }
276
277 for (Track t : presentationTrackSelector.select(mediaPackage, false)) {
278 logger.info("Found partial presentation track {}", t);
279 originalTracks.add(t);
280 }
281
282
283 logger.info("Starting preencoding");
284 originalTracks = preencode(preencodeProfile, originalTracks);
285
286
287
288 final Map<String, Job> jobs = new HashMap<String, Job>();
289
290 final SMILDocument smilDocument;
291 try {
292 smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
293 } catch (SAXException e) {
294 throw new WorkflowOperationException(e);
295 }
296 final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
297 final NodeList sequences = parallel.getTimeChildren();
298 final float trackDurationInSeconds = parallel.getDur();
299 final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
300 for (int i = 0; i < sequences.getLength(); i++) {
301 final SMILElement item = (SMILElement) sequences.item(i);
302
303 for (final String mediaType : new String[] { NODE_TYPE_AUDIO, NODE_TYPE_VIDEO }) {
304 final List<Track> tracks = new ArrayList<Track>();
305 final VCell<String> sourceType = VCell.cell(EMPTY_VALUE);
306
307 final long position = processChildren(0, tracks, item.getChildNodes(), originalTracks, sourceType, mediaType,
308 elementsToClean, operationId);
309
310 if (tracks.isEmpty()) {
311 logger.debug("The tracks list was empty.");
312 continue;
313 }
314 final Track lastTrack = tracks.get(tracks.size() - 1);
315
316 if (position < trackDurationInMs) {
317 final double extendingTime = (trackDurationInMs - position) / 1000d;
318 if (extendingTime > 0) {
319 if (!lastTrack.hasVideo()) {
320 logger.info("Extending {} audio track end by {} seconds with silent audio", sourceType.get(),
321 extendingTime);
322 tracks.add(getSilentAudio(extendingTime, elementsToClean, operationId));
323 } else {
324 logger.info("Extending {} track end with last image frame by {} seconds", sourceType.get(), extendingTime);
325 Attachment tempLastImageFrame = extractLastImageFrame(lastTrack, elementsToClean);
326 tracks.add(createVideoFromImage(tempLastImageFrame, extendingTime, elementsToClean));
327 }
328 }
329 }
330
331 if (tracks.size() < 2) {
332 logger.debug("There were less than 2 tracks, copying track...");
333 if (sourceType.get().startsWith(PRESENTER_KEY)) {
334 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresenterFlavor);
335 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
336 createCopyOfTrack(mediaPackage, tracks.get(0), targetPresentationFlavor);
337 } else {
338 logger.warn("Can't handle unkown source type '{}' for unprocessed track", sourceType.get());
339 }
340 continue;
341 }
342
343 for (final Track t : tracks) {
344 if (!t.hasVideo() && !t.hasAudio()) {
345 logger.error("No audio or video stream available in the track with flavor {}! {}", t.getFlavor(), t);
346 throw new WorkflowOperationException("No audio or video stream available in the track " + t.toString());
347 }
348 }
349
350 if (sourceType.get().startsWith(PRESENTER_KEY)) {
351 logger.info("Concatenating {} track", PRESENTER_KEY);
352 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
353 } else if (sourceType.get().startsWith(PRESENTATION_KEY)) {
354 logger.info("Concatenating {} track", PRESENTATION_KEY);
355 jobs.put(sourceType.get(), startConcatJob(concatProfile, tracks, outputFramerate, forceDivisible));
356 } else {
357 logger.warn("Can't handle unknown source type '{}'!", sourceType.get());
358 }
359 }
360 }
361
362
363 if (jobs.size() > 0) {
364 if (!JobUtil.waitForJobs(serviceRegistry, jobs.values()).isSuccess()) {
365 throw new WorkflowOperationException("One of the concat jobs did not complete successfully");
366 }
367 } else {
368 logger.info("No concatenating needed for presenter and presentation tracks, took partial source elements");
369 }
370
371
372 long queueTime = 0L;
373 MediaPackageElementFlavor adjustedTargetPresenterFlavor = targetPresenterFlavor;
374 MediaPackageElementFlavor adjustedTargetPresentationFlavor = targetPresentationFlavor;
375 for (final Entry<String, Job> job : jobs.entrySet()) {
376 final Optional<Job> concatJob = Optional.of(JobUtil.update(serviceRegistry, job.getValue()).orNull());
377 if (concatJob.isPresent()) {
378 final String concatPayload = concatJob.get().getPayload();
379 if (concatPayload != null) {
380 final Track concatTrack;
381 try {
382 concatTrack = (Track) MediaPackageElementParser.getFromXml(concatPayload);
383 } catch (MediaPackageException e) {
384 throw new WorkflowOperationException(e);
385 }
386
387 final String fileName;
388
389
390 if (job.getKey().startsWith(PRESENTER_KEY)) {
391 if (!concatTrack.hasVideo()) {
392 fileName = PRESENTER_KEY.concat(FLAVOR_AUDIO_SUFFIX);
393 adjustedTargetPresenterFlavor = deriveAudioFlavor(targetPresenterFlavor);
394 } else {
395 fileName = PRESENTER_KEY;
396 adjustedTargetPresenterFlavor = targetPresenterFlavor;
397 }
398 concatTrack.setFlavor(adjustedTargetPresenterFlavor);
399 } else if (job.getKey().startsWith(PRESENTATION_KEY)) {
400 if (!concatTrack.hasVideo()) {
401 fileName = PRESENTATION_KEY.concat(FLAVOR_AUDIO_SUFFIX);
402 adjustedTargetPresentationFlavor = deriveAudioFlavor(targetPresentationFlavor);
403 } else {
404 fileName = PRESENTATION_KEY;
405 adjustedTargetPresentationFlavor = targetPresentationFlavor;
406 }
407 concatTrack.setFlavor(adjustedTargetPresentationFlavor);
408 } else {
409 fileName = UNKNOWN_KEY;
410 }
411
412 concatTrack.setURI(workspace.moveTo(concatTrack.getURI(), mediaPackage.getIdentifier().toString(),
413 concatTrack.getIdentifier(),
414 fileName + "." + FilenameUtils.getExtension(concatTrack.getURI().toString())));
415
416 logger.info("Concatenated track {} got flavor '{}'", concatTrack, concatTrack.getFlavor());
417
418 mediaPackage.add(concatTrack);
419 queueTime += concatJob.get().getQueueTime();
420 } else {
421
422 logger.warn("Concat job {} does not contain a payload", concatJob);
423 }
424 } else {
425 logger.warn("Concat job {} could not be updated since it cannot be found", job.getValue());
426 }
427 }
428
429
430 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresentationFlavor, trackDurationInSeconds,
431 elementsToClean);
432 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresentationFlavor),
433 trackDurationInSeconds, elementsToClean);
434 queueTime += checkForTrimming(mediaPackage, trimProfile, targetPresenterFlavor, trackDurationInSeconds,
435 elementsToClean);
436 queueTime += checkForTrimming(mediaPackage, trimProfile, deriveAudioFlavor(targetPresenterFlavor),
437 trackDurationInSeconds, elementsToClean);
438
439
440 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, deriveAudioFlavor(targetPresenterFlavor), false, elementsToClean);
441 queueTime += checkForMuxing(mediaPackage, targetPresentationFlavor, deriveAudioFlavor(targetPresentationFlavor), false, elementsToClean);
442
443 adjustAudioTrackTargetFlavor(mediaPackage, targetPresenterFlavor);
444 adjustAudioTrackTargetFlavor(mediaPackage, targetPresentationFlavor);
445
446
447 queueTime += checkForMuxing(mediaPackage, targetPresenterFlavor, targetPresentationFlavor, false, elementsToClean);
448
449 queueTime += checkForEncodeToStandard(mediaPackage, forceEncoding, forceProfile, requiredExtensions,
450 targetPresenterFlavor, targetPresentationFlavor, elementsToClean);
451
452 final WorkflowOperationResult result = createResult(mediaPackage, Action.CONTINUE, queueTime);
453 logger.debug("Partial import operation completed");
454 return result;
455 }
456
457 protected long checkForEncodeToStandard(MediaPackage mediaPackage, boolean forceEncoding,
458 Optional<EncodingProfile> forceProfile, List<String> requiredExtensions,
459 MediaPackageElementFlavor targetPresenterFlavor, MediaPackageElementFlavor targetPresentationFlavor,
460 List<MediaPackageElement> elementsToClean) throws EncoderException, IOException, MediaPackageException,
461 NotFoundException, ServiceRegistryException, WorkflowOperationException {
462 long queueTime = 0;
463 if (forceProfile.isPresent()) {
464 Track[] targetPresenterTracks = mediaPackage.getTracks(targetPresenterFlavor);
465 for (Track track : targetPresenterTracks) {
466 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
467 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
468 targetPresenterFlavor, track.getURI(), forceProfile.get());
469 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresenterFlavor, track);
470 elementsToClean.add(track);
471 mediaPackage.remove(track);
472 }
473 }
474
475 if (!targetPresenterFlavor.toString().equalsIgnoreCase(targetPresentationFlavor.toString())) {
476 Track[] targetPresentationTracks = mediaPackage.getTracks(targetPresentationFlavor);
477 for (Track track : targetPresentationTracks) {
478 if (forceEncoding || trackNeedsTobeEncodedToStandard(track, requiredExtensions)) {
479 logger.debug("Encoding '{}' flavored track '{}' with standard encoding profile {}",
480 targetPresentationFlavor, track.getURI(), forceProfile.get());
481 queueTime += encodeToStandard(mediaPackage, forceProfile.get(), targetPresentationFlavor, track);
482 elementsToClean.add(track);
483 mediaPackage.remove(track);
484 }
485 }
486 }
487 }
488 return queueTime;
489 }
490
491
492
493
494
495
496
497
498
499
500
501 private void createCopyOfTrack(MediaPackage mediaPackage, Track track, MediaPackageElementFlavor targetFlavor)
502 throws IllegalArgumentException, NotFoundException,IOException {
503
504 MediaPackageElementFlavor targetCopyFlavor = null;
505 if (track.hasVideo()) {
506 targetCopyFlavor = targetFlavor;
507 } else {
508 targetCopyFlavor = deriveAudioFlavor(targetFlavor);
509 }
510 logger.debug("Copying track {} with flavor {} using target flavor {}", track.getURI(), track.getFlavor(), targetCopyFlavor);
511 copyPartialToSource(mediaPackage, targetCopyFlavor, track);
512 }
513
514
515
516
517
518
519
520
521
522
523
524
525 private void adjustAudioTrackTargetFlavor(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor)
526 throws IllegalArgumentException, NotFoundException,IOException {
527
528 Track[] targetAudioTracks = mediaPackage.getTracks(deriveAudioFlavor(targetFlavor));
529 for (Track track : targetAudioTracks) {
530 logger.debug("Adding {} to finished audio tracks.", track.getURI());
531 mediaPackage.remove(track);
532 track.setFlavor(targetFlavor);
533 mediaPackage.add(track);
534 }
535 }
536
537 private TrackSelector mkTrackSelector(Optional<String> flavor) throws WorkflowOperationException {
538 final TrackSelector s = new TrackSelector();
539 if (flavor.isPresent()) {
540 try {
541 final MediaPackageElementFlavor f = MediaPackageElementFlavor.parseFlavor(flavor.get());
542 s.addFlavor(f);
543 s.addFlavor(deriveAudioFlavor(f));
544 } catch (IllegalArgumentException e) {
545 throw new WorkflowOperationException("Flavor '" + flavor.get() + "' is malformed");
546 }
547 }
548 return s;
549 }
550
551
552
553
554
555
556
557
558
559
560
561 protected Job startConcatJob(EncodingProfile profile, List<Track> tracks, float outputFramerate, boolean forceDivisible)
562 throws MediaPackageException, EncoderException {
563 final Dimension dim = determineDimension(tracks, forceDivisible);
564 if (outputFramerate > 0.0) {
565 return composerService.concat(profile.getIdentifier(), dim, outputFramerate, true, Collections.toArray(Track.class, tracks));
566 } else {
567 return composerService.concat(profile.getIdentifier(), dim, true, Collections.toArray(Track.class, tracks)); }
568 }
569
570
571
572
573
574
575
576 protected static boolean trackNeedsTobeEncodedToStandard(Track track, List<String> requiredExtensions) {
577 String extension = FilenameUtils.getExtension(track.getURI().toString());
578 for (String requiredExtension : requiredExtensions) {
579 if (requiredExtension.equalsIgnoreCase(extension)) {
580 return false;
581 }
582 }
583 return true;
584 }
585
586
587
588
589
590
591
592
593 protected List<String> getRequiredExtensions(WorkflowOperationInstance operation) {
594 List<String> requiredExtensions = new ArrayList<String>();
595 String configExtensions = null;
596 try {
597 configExtensions = StringUtils.trimToNull(getConfig(operation, REQUIRED_EXTENSIONS));
598 } catch (WorkflowOperationException e) {
599 logger.info(
600 "Required extensions configuration key not specified so will be using default '{}'. Any input file not matching this extension will be re-encoded.",
601 DEFAULT_REQUIRED_EXTENSION);
602 }
603 if (configExtensions != null) {
604 String[] extensions = configExtensions.split(",");
605 for (String extension : extensions) {
606 requiredExtensions.add(extension);
607 }
608 }
609 if (requiredExtensions.size() == 0) {
610 requiredExtensions.add(DEFAULT_REQUIRED_EXTENSION);
611 }
612 return requiredExtensions;
613 }
614
615
616
617
618
619
620
621
622 protected Optional<EncodingProfile> getForceEncodingProfile(WorkflowOperationInstance woi, boolean forceEncoding)
623 throws WorkflowOperationException {
624 if (!forceEncoding) {
625 return Optional.empty();
626 }
627
628 Optional<String> profileNameOpt = Optional.ofNullable(getOptConfig(woi, FORCE_ENCODING_PROFILE).orNull());
629 if (forceEncoding && profileNameOpt.isEmpty()) {
630 throw new WorkflowOperationException("Force encoding profile must be set!");
631 }
632
633 String profileName = profileNameOpt.get();
634 EncodingProfile profile = composerService.getProfile(profileName);
635 if (profile == null) {
636 throw new WorkflowOperationException("Force encoding profile '" + profileName + "' was not found");
637 }
638
639 return Optional.of(profile);
640 }
641
642
643
644
645
646 private MediaPackageElementFlavor parseTargetFlavor(String flavor, String flavorType)
647 throws WorkflowOperationException {
648 final MediaPackageElementFlavor targetFlavor;
649 try {
650 targetFlavor = MediaPackageElementFlavor.parseFlavor(flavor);
651 if ("*".equals(targetFlavor.getType()) || "*".equals(targetFlavor.getSubtype())) {
652 throw new WorkflowOperationException(format(
653 "Target %s flavor must have a type and a subtype, '*' are not allowed!", flavorType));
654 }
655 } catch (IllegalArgumentException e) {
656 throw new WorkflowOperationException(format("Target %s flavor '%s' is malformed", flavorType, flavor));
657 }
658 return targetFlavor;
659 }
660
661
662 private MediaPackageElementFlavor deriveAudioFlavor(MediaPackageElementFlavor flavor) {
663 return MediaPackageElementFlavor.flavor(flavor.getType().concat(FLAVOR_AUDIO_SUFFIX), flavor.getSubtype());
664 }
665
666
667
668
669
670
671
672
673
674
675 private Dimension determineDimension(List<Track> tracks, boolean forceDivisible) {
676 Tuple<Track, Dimension> trackDimension = getLargestTrack(tracks);
677 if (trackDimension == null)
678 return null;
679
680 if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
681 Dimension scaledDimension = Dimension.dimension((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
682 .getB().getHeight() / 2) * 2);
683 logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
684 trackDimension.getB(), trackDimension.getA());
685 return scaledDimension;
686 } else {
687 logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
688 return trackDimension.getB();
689 }
690 }
691
692
693
694
695
696
697
698
699 private Tuple<Track, Dimension> getLargestTrack(List<Track> tracks) {
700 Track track = null;
701 Dimension dimension = null;
702 for (Track t : tracks) {
703 if (!t.hasVideo())
704 continue;
705
706 VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
707 int frameWidth = videoStreams[0].getFrameWidth();
708 int frameHeight = videoStreams[0].getFrameHeight();
709 if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
710 dimension = Dimension.dimension(frameWidth, frameHeight);
711 track = t;
712 }
713 }
714 if (track == null || dimension == null)
715 return null;
716
717 return Tuple.tuple(track, dimension);
718 }
719
720 private long checkForTrimming(MediaPackage mediaPackage, EncodingProfile trimProfile,
721 MediaPackageElementFlavor targetFlavor, Float videoDuration, List<MediaPackageElement> elementsToClean)
722 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
723 ServiceRegistryException, IOException {
724 MediaPackageElement[] elements = mediaPackage.getElementsByFlavor(targetFlavor);
725 if (elements.length == 0)
726 return 0;
727
728 Track trackToTrim = (Track) elements[0];
729 if (elements.length == 1 && trackToTrim.getDuration() / 1000 > videoDuration) {
730 Long trimSeconds = (long) (trackToTrim.getDuration() / 1000 - videoDuration);
731 logger.info("Shorten track {} to target duration {} by {} seconds",
732 trackToTrim.toString(), videoDuration.toString(), trimSeconds.toString());
733 return trimEnd(mediaPackage, trimProfile, trackToTrim, videoDuration, elementsToClean);
734 } else if (elements.length > 1) {
735 logger.warn("Multiple tracks with flavor {} found! Trimming not possible!", targetFlavor);
736 }
737 return 0;
738 }
739
740 private List<Track> getPureVideoTracks(MediaPackage mediaPackage, MediaPackageElementFlavor videoFlavor) {
741 return Arrays.stream(mediaPackage.getTracks())
742 .filter(track -> track.getFlavor().matches(videoFlavor))
743 .filter(Track::hasVideo)
744 .filter(track -> !track.hasAudio())
745 .collect(Collectors.toList());
746 }
747
748 private List<Track> getPureAudioTracks(MediaPackage mediaPackage, MediaPackageElementFlavor audioFlavor) {
749 return Arrays.stream(mediaPackage.getTracks())
750 .filter(track -> track.getFlavor().matches(audioFlavor))
751 .filter(Track::hasAudio)
752 .filter(track -> !track.hasVideo())
753 .collect(Collectors.toList());
754 }
755
756 protected long checkForMuxing(MediaPackage mediaPackage, MediaPackageElementFlavor targetPresentationFlavor,
757 MediaPackageElementFlavor targetPresenterFlavor, boolean useSuffix, List<MediaPackageElement> elementsToClean)
758 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
759 ServiceRegistryException, IOException {
760
761 long queueTime = 0L;
762
763 List<Track> videoElements = getPureVideoTracks(mediaPackage, targetPresentationFlavor);
764 List<Track> audioElements;
765 if (useSuffix) {
766 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresentationFlavor));
767 } else {
768 audioElements = getPureAudioTracks(mediaPackage, targetPresentationFlavor);
769 }
770
771 Track videoTrack = null;
772 Track audioTrack = null;
773
774 if (videoElements.size() == 1 && audioElements.size() == 0) {
775 videoTrack = videoElements.get(0);
776 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
777 audioTrack = audioElements.get(0);
778 }
779
780 videoElements = getPureVideoTracks(mediaPackage, targetPresenterFlavor);
781 if (useSuffix) {
782 audioElements = getPureAudioTracks(mediaPackage, deriveAudioFlavor(targetPresenterFlavor));
783 } else {
784 audioElements = getPureAudioTracks(mediaPackage, targetPresenterFlavor);
785 }
786
787 if (videoElements.size() == 1 && audioElements.size() == 0) {
788 videoTrack = videoElements.get(0);
789 } else if (videoElements.size() == 0 && audioElements.size() == 1) {
790 audioTrack = audioElements.get(0);
791 }
792
793 logger.debug("Check for mux between '{}' and '{}' flavors and found video track '{}' and audio track '{}'",
794 targetPresentationFlavor, targetPresenterFlavor, videoTrack, audioTrack);
795 if (videoTrack != null && audioTrack != null) {
796 queueTime += mux(mediaPackage, videoTrack, audioTrack, elementsToClean);
797 return queueTime;
798 } else {
799 return queueTime;
800 }
801 }
802
803
804
805
806
807
808
809 protected long mux(MediaPackage mediaPackage, Track video, Track audio, List<MediaPackageElement> elementsToClean)
810 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
811 ServiceRegistryException, IOException {
812 logger.debug("Muxing video {} and audio {}", video.getURI(), audio.getURI());
813 Job muxJob = composerService.mux(video, audio, PrepareAVWorkflowOperationHandler.MUX_AV_PROFILE);
814 if (!waitForStatus(muxJob).isSuccess()) {
815 throw new WorkflowOperationException("Muxing of audio " + audio + " and video " + video + " failed");
816 }
817 muxJob = serviceRegistry.getJob(muxJob.getId());
818
819 final Track muxed = (Track) MediaPackageElementParser.getFromXml(muxJob.getPayload());
820 if (muxed == null) {
821 throw new WorkflowOperationException("Muxed job " + muxJob + " returned no payload!");
822 }
823 muxed.setFlavor(video.getFlavor());
824 muxed.setURI(workspace.moveTo(muxed.getURI(), mediaPackage.getIdentifier().toString(), muxed.getIdentifier(),
825 FilenameUtils.getName(video.getURI().toString())));
826 elementsToClean.add(audio);
827 mediaPackage.remove(audio);
828 elementsToClean.add(video);
829 mediaPackage.remove(video);
830 mediaPackage.add(muxed);
831 return muxJob.getQueueTime();
832 }
833
834 private void copyPartialToSource(MediaPackage mediaPackage, MediaPackageElementFlavor targetFlavor, Track track)
835 throws NotFoundException, IOException {
836 FileInputStream in = null;
837 try {
838 Track copyTrack = (Track) track.clone();
839 File originalFile = workspace.get(copyTrack.getURI());
840 in = new FileInputStream(originalFile);
841
842 String elementID = UUID.randomUUID().toString();
843 copyTrack.setURI(workspace.put(mediaPackage.getIdentifier().toString(), elementID,
844 FilenameUtils.getName(copyTrack.getURI().toString()), in));
845 copyTrack.setFlavor(targetFlavor);
846 copyTrack.setIdentifier(elementID);
847 copyTrack.referTo(track);
848 mediaPackage.add(copyTrack);
849 logger.info("Copied partial source element {} to {} with target flavor {}", track.toString(),
850 copyTrack.toString(), targetFlavor.toString());
851 } finally {
852 IOUtils.closeQuietly(in);
853 }
854 }
855
856
857
858
859
860
861
862
863 private List<Track> preencode(EncodingProfile profile, List<Track> tracks)
864 throws MediaPackageException, EncoderException, WorkflowOperationException, NotFoundException,
865 ServiceRegistryException {
866 List<Track> encodedTracks = new ArrayList<>();
867 for (Track track : tracks) {
868 logger.info("Preencoding track {}", track.getIdentifier());
869 Job encodeJob = composerService.encode(track, profile.getIdentifier());
870 if (!waitForStatus(encodeJob).isSuccess()) {
871 throw new WorkflowOperationException("Encoding of track " + track + " failed");
872 }
873 encodeJob = serviceRegistry.getJob(encodeJob.getId());
874 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
875 if (encodedTrack == null) {
876 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
877 }
878 encodedTrack.setIdentifier(track.getIdentifier());
879 encodedTrack.setFlavor(track.getFlavor());
880 encodedTracks.add(encodedTrack);
881 }
882
883 return encodedTracks;
884 }
885
886
887
888
889
890
891
892 private long encodeToStandard(MediaPackage mp, EncodingProfile profile, MediaPackageElementFlavor targetFlavor,
893 Track track) throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException,
894 ServiceRegistryException, IOException {
895 Job encodeJob = composerService.encode(track, profile.getIdentifier());
896 if (!waitForStatus(encodeJob).isSuccess()) {
897 throw new WorkflowOperationException("Encoding of track " + track + " failed");
898 }
899 encodeJob = serviceRegistry.getJob(encodeJob.getId());
900 Track encodedTrack = (Track) MediaPackageElementParser.getFromXml(encodeJob.getPayload());
901 if (encodedTrack == null) {
902 throw new WorkflowOperationException("Encoded track " + track + " failed to produce a track");
903 }
904 URI uri;
905 if (FilenameUtils.getExtension(encodedTrack.getURI().toString()).equalsIgnoreCase(
906 FilenameUtils.getExtension(track.getURI().toString()))) {
907 uri = workspace.moveTo(encodedTrack.getURI(), mp.getIdentifier().toString(), encodedTrack.getIdentifier(),
908 FilenameUtils.getName(track.getURI().toString()));
909 } else {
910
911 uri = workspace.moveTo(
912 encodedTrack.getURI(),
913 mp.getIdentifier().toString(),
914 encodedTrack.getIdentifier(),
915 FilenameUtils.getBaseName(track.getURI().toString()) + "."
916 + FilenameUtils.getExtension(encodedTrack.getURI().toString()));
917 }
918 encodedTrack.setURI(uri);
919 encodedTrack.setFlavor(targetFlavor);
920 mp.add(encodedTrack);
921 return encodeJob.getQueueTime();
922 }
923
924 private long trimEnd(MediaPackage mediaPackage, EncodingProfile trimProfile, Track track, double duration,
925 List<MediaPackageElement> elementsToClean) throws EncoderException, MediaPackageException,
926 WorkflowOperationException, NotFoundException, ServiceRegistryException, IOException {
927 Job trimJob = composerService.trim(track, trimProfile.getIdentifier(), 0, (long) (duration * 1000));
928 if (!waitForStatus(trimJob).isSuccess())
929 throw new WorkflowOperationException("Trimming of track " + track + " failed");
930
931 trimJob = serviceRegistry.getJob(trimJob.getId());
932
933 Track trimmedTrack = (Track) MediaPackageElementParser.getFromXml(trimJob.getPayload());
934 if (trimmedTrack == null)
935 throw new WorkflowOperationException("Trimming track " + track + " failed to produce a track");
936
937 URI uri = workspace.moveTo(trimmedTrack.getURI(), mediaPackage.getIdentifier().toString(),
938 trimmedTrack.getIdentifier(), FilenameUtils.getName(track.getURI().toString()));
939 trimmedTrack.setURI(uri);
940 trimmedTrack.setFlavor(track.getFlavor());
941
942 elementsToClean.add(track);
943 mediaPackage.remove(track);
944 mediaPackage.add(trimmedTrack);
945
946 return trimJob.getQueueTime();
947 }
948
949 private long processChildren(long position, List<Track> tracks, NodeList children, List<Track> originalTracks,
950 VCell<String> type, String mediaType, List<MediaPackageElement> elementsToClean, Long operationId)
951 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException, IOException {
952 for (int j = 0; j < children.getLength(); j++) {
953 Node item = children.item(j);
954 if (item.hasChildNodes()) {
955 position = processChildren(position, tracks, item.getChildNodes(), originalTracks, type, mediaType,
956 elementsToClean, operationId);
957 } else {
958 SMILMediaElement e = (SMILMediaElement) item;
959 if (mediaType.equals(e.getNodeName())) {
960 Track track;
961 try {
962 track = getFromOriginal(e.getId(), originalTracks, type);
963 } catch (IllegalStateException exception) {
964 logger.debug("Skipping smil entry, reason: " + exception.getMessage());
965 continue;
966 }
967 double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
968 long beginInMs = Math.round(beginInSeconds * 1000d);
969
970 if (beginInMs > position) {
971 double positionInSeconds = position / 1000d;
972 if (position == 0) {
973 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
974 logger.info("Extending {} audio track start by {} seconds silent audio", type.get(), beginInSeconds);
975 tracks.add(getSilentAudio(beginInSeconds, elementsToClean, operationId));
976 } else {
977 logger.info("Extending {} track start image frame by {} seconds", type.get(), beginInSeconds);
978 Attachment tempFirstImageFrame = extractImage(track, 0, elementsToClean);
979 tracks.add(createVideoFromImage(tempFirstImageFrame, beginInSeconds, elementsToClean));
980 }
981 position += beginInMs;
982 } else {
983 double fillTime = (beginInMs - position) / 1000d;
984 if (NODE_TYPE_AUDIO.equals(e.getNodeName())) {
985 logger.info("Fill {} audio track gap from {} to {} with silent audio", type.get(),
986 Double.toString(positionInSeconds), Double.toString(beginInSeconds));
987 tracks.add(getSilentAudio(fillTime, elementsToClean, operationId));
988 } else {
989 logger.info("Fill {} track gap from {} to {} with image frame",
990 type.get(), Double.toString(positionInSeconds), Double.toString(beginInSeconds));
991 Track previousTrack = tracks.get(tracks.size() - 1);
992 Attachment tempLastImageFrame = extractLastImageFrame(previousTrack, elementsToClean);
993 tracks.add(createVideoFromImage(tempLastImageFrame, fillTime, elementsToClean));
994 }
995 position = beginInMs;
996 }
997 }
998 tracks.add(track);
999 position += Math.round(e.getDur() * 1000f);
1000 }
1001 }
1002 }
1003 return position;
1004 }
1005
1006 private Track getFromOriginal(String trackId, List<Track> originalTracks, VCell<String> type) {
1007 for (Track t : originalTracks) {
1008 if (t.getIdentifier().contains(trackId)) {
1009 logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
1010 if (EMPTY_VALUE.equals(type.get())) {
1011 String suffix = (t.hasAudio() && !t.hasVideo()) ? FLAVOR_AUDIO_SUFFIX : "";
1012 type.set(t.getFlavor().getType() + suffix);
1013 }
1014 originalTracks.remove(t);
1015 return t;
1016 }
1017 }
1018 throw new IllegalStateException("No track matching smil Track-id: " + trackId);
1019 }
1020
1021 private Track getSilentAudio(final double time, final List<MediaPackageElement> elementsToClean,
1022 final Long operationId) throws EncoderException, MediaPackageException, WorkflowOperationException,
1023 NotFoundException, IOException {
1024 final URI uri = workspace.putInCollection(COLLECTION_ID, operationId + "-silent", new ByteArrayInputStream(
1025 EMPTY_VALUE.getBytes()));
1026 final Attachment emptyAttachment = (Attachment) MediaPackageElementBuilderFactory.newInstance().newElementBuilder()
1027 .elementFromURI(uri, Type.Attachment, MediaPackageElementFlavor.parseFlavor("audio/silent"));
1028 elementsToClean.add(emptyAttachment);
1029
1030 final Job silentAudioJob = composerService.imageToVideo(emptyAttachment, SILENT_AUDIO_PROFILE, time);
1031 if (!waitForStatus(silentAudioJob).isSuccess())
1032 throw new WorkflowOperationException("Silent audio job did not complete successfully");
1033
1034
1035 try {
1036 for (final String payload : getPayload(serviceRegistry, silentAudioJob)) {
1037 final Track silentAudio = (Track) MediaPackageElementParser.getFromXml(payload);
1038 elementsToClean.add(silentAudio);
1039 return silentAudio;
1040 }
1041
1042 throw new WorkflowOperationException(format("Job %s has no payload or cannot be updated", silentAudioJob));
1043 } catch (ServiceRegistryException ex) {
1044 throw new WorkflowOperationException(ex);
1045 }
1046 }
1047
1048 private Track createVideoFromImage(Attachment image, double time, List<MediaPackageElement> elementsToClean)
1049 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1050 Job imageToVideoJob = composerService.imageToVideo(image, IMAGE_MOVIE_PROFILE, time);
1051 if (!waitForStatus(imageToVideoJob).isSuccess())
1052 throw new WorkflowOperationException("Image to video job did not complete successfully");
1053
1054
1055 try {
1056 imageToVideoJob = serviceRegistry.getJob(imageToVideoJob.getId());
1057 } catch (ServiceRegistryException e) {
1058 throw new WorkflowOperationException(e);
1059 }
1060 Track imageVideo = (Track) MediaPackageElementParser.getFromXml(imageToVideoJob.getPayload());
1061 elementsToClean.add(imageVideo);
1062 return imageVideo;
1063 }
1064
1065 private Attachment extractImage(Track presentationTrack, double time, List<MediaPackageElement> elementsToClean)
1066 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1067 Job extractImageJob = composerService.image(presentationTrack, PREVIEW_PROFILE, time);
1068 if (!waitForStatus(extractImageJob).isSuccess())
1069 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1070
1071
1072 try {
1073 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1074 } catch (ServiceRegistryException e) {
1075 throw new WorkflowOperationException(e);
1076 }
1077 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1078 .get(0);
1079 elementsToClean.add(composedImages);
1080 return composedImages;
1081 }
1082
1083 private Attachment extractLastImageFrame(Track presentationTrack, List<MediaPackageElement> elementsToClean)
1084 throws EncoderException, MediaPackageException, WorkflowOperationException, NotFoundException {
1085
1086 Map<String, String> properties = new HashMap<String, String>();
1087
1088 Job extractImageJob = composerService.image(presentationTrack, IMAGE_FRAME_PROFILE, properties);
1089 if (!waitForStatus(extractImageJob).isSuccess())
1090 throw new WorkflowOperationException("Extract image frame video job did not complete successfully");
1091
1092
1093 try {
1094 extractImageJob = serviceRegistry.getJob(extractImageJob.getId());
1095 } catch (ServiceRegistryException e) {
1096 throw new WorkflowOperationException(e);
1097 }
1098 Attachment composedImages = (Attachment) MediaPackageElementParser.getArrayFromXml(extractImageJob.getPayload())
1099 .get(0);
1100 elementsToClean.add(composedImages);
1101 return composedImages;
1102 }
1103 }