1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.workflow.handler.videogrid;
22
23 import static java.lang.String.format;
24
25 import org.opencastproject.composer.api.ComposerService;
26 import org.opencastproject.composer.api.EncoderException;
27 import org.opencastproject.composer.api.EncodingProfile;
28 import org.opencastproject.composer.layout.Dimension;
29 import org.opencastproject.inspection.api.MediaInspectionException;
30 import org.opencastproject.inspection.api.MediaInspectionService;
31 import org.opencastproject.job.api.Job;
32 import org.opencastproject.job.api.JobContext;
33 import org.opencastproject.mediapackage.MediaPackage;
34 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
35 import org.opencastproject.mediapackage.MediaPackageElementParser;
36 import org.opencastproject.mediapackage.MediaPackageException;
37 import org.opencastproject.mediapackage.Track;
38 import org.opencastproject.mediapackage.TrackSupport;
39 import org.opencastproject.mediapackage.VideoStream;
40 import org.opencastproject.mediapackage.selector.TrackSelector;
41 import org.opencastproject.mediapackage.track.TrackImpl;
42 import org.opencastproject.serviceregistry.api.ServiceRegistry;
43 import org.opencastproject.smil.api.util.SmilUtil;
44 import org.opencastproject.util.NotFoundException;
45 import org.opencastproject.util.data.Tuple;
46 import org.opencastproject.videogrid.api.VideoGridService;
47 import org.opencastproject.videogrid.api.VideoGridServiceException;
48 import org.opencastproject.workflow.api.AbstractWorkflowOperationHandler;
49 import org.opencastproject.workflow.api.ConfiguredTagsAndFlavors;
50 import org.opencastproject.workflow.api.WorkflowInstance;
51 import org.opencastproject.workflow.api.WorkflowOperationException;
52 import org.opencastproject.workflow.api.WorkflowOperationHandler;
53 import org.opencastproject.workflow.api.WorkflowOperationInstance;
54 import org.opencastproject.workflow.api.WorkflowOperationResult;
55 import org.opencastproject.workspace.api.Workspace;
56
57 import com.google.gson.Gson;
58 import com.google.gson.reflect.TypeToken;
59
60 import org.apache.commons.lang3.StringUtils;
61 import org.apache.commons.lang3.tuple.ImmutablePair;
62 import org.osgi.service.component.annotations.Component;
63 import org.osgi.service.component.annotations.Reference;
64 import org.slf4j.Logger;
65 import org.slf4j.LoggerFactory;
66 import org.w3c.dom.Node;
67 import org.w3c.dom.NodeList;
68 import org.w3c.dom.smil.SMILDocument;
69 import org.w3c.dom.smil.SMILElement;
70 import org.w3c.dom.smil.SMILMediaElement;
71 import org.w3c.dom.smil.SMILParElement;
72 import org.xml.sax.SAXException;
73
74 import java.io.File;
75 import java.io.IOException;
76 import java.net.URI;
77 import java.util.ArrayList;
78 import java.util.Arrays;
79 import java.util.Collections;
80 import java.util.HashMap;
81 import java.util.List;
82 import java.util.Locale;
83 import java.util.Map;
84 import java.util.regex.Pattern;
85 import java.util.stream.Collectors;
86
87
88
89
90
91
92
93
94
95
96
97 @Component(
98 immediate = true,
99 service = WorkflowOperationHandler.class,
100 property = {
101 "service.description=Video Grid Workflow Operation Handler",
102 "workflow.operation=videogrid"
103 }
104 )
105 public class VideoGridWorkflowOperationHandler extends AbstractWorkflowOperationHandler {
106
107
108 private static final String SOURCE_FLAVORS = "source-flavors";
109 private static final String SOURCE_SMIL_FLAVOR = "source-smil-flavor";
110 private static final String CONCAT_ENCODING_PROFILE = "concat-encoding-profile";
111
112 private static final String OPT_RESOLUTION = "resolution";
113 private static final String OPT_BACKGROUND_COLOR = "background-color";
114
115
116 private static final Logger logger = LoggerFactory.getLogger(VideoGridWorkflowOperationHandler.class);
117
118
119 private static final String NODE_TYPE_VIDEO = "video";
120
121
122 private static final String[] FFMPEG = {"ffmpeg", "-y", "-v", "warning", "-nostats", "-max_error_rate", "1.0"};
123 private static final String FFMPEG_WF_CODEC = "h264";
124 private static final int FFMPEG_WF_FRAMERATE = 24;
125 private static final String[] FFMPEG_WF_ARGS = {
126 "-an", "-codec", FFMPEG_WF_CODEC,
127 "-q:v", "2",
128 "-g", Integer.toString(FFMPEG_WF_FRAMERATE * 10),
129 "-pix_fmt", "yuv420p",
130 "-r", Integer.toString(FFMPEG_WF_FRAMERATE)
131 };
132
133
134 private Workspace workspace = null;
135 private VideoGridService videoGridService = null;
136 private MediaInspectionService inspectionService = null;
137 private ComposerService composerService = null;
138
139
140 @Reference
141 public void setWorkspace(Workspace workspace) {
142 this.workspace = workspace;
143 }
144 @Reference
145 public void setVideoGridService(VideoGridService videoGridService) {
146 this.videoGridService = videoGridService;
147 }
148 @Reference
149 protected void setMediaInspectionService(MediaInspectionService inspectionService) {
150 this.inspectionService = inspectionService;
151 }
152 @Reference
153 public void setComposerService(ComposerService composerService) {
154 this.composerService = composerService;
155 }
156 @Reference
157 @Override
158 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
159 super.setServiceRegistry(serviceRegistry);
160 }
161
162
163
164
165
166
167 class LayoutArea {
168 private int x = 0;
169 private int y = 0;
170 private int width = 1920;
171 private int height = 1080;
172 private String name = "webcam";
173 private String bgColor = "0xFFFFFF";
174
175 public int getX() {
176 return x;
177 }
178 public void setX(int x) {
179 this.x = x;
180 }
181 public int getY() {
182 return y;
183 }
184 public void setY(int y) {
185 this.y = y;
186 }
187 public int getWidth() {
188 return width;
189 }
190 public void setWidth(int width) {
191 this.width = width;
192 }
193 public int getHeight() {
194 return height;
195 }
196 public void setHeight(int height) {
197 this.height = height;
198 }
199 public String getName() {
200 return name;
201 }
202 public void setName(String name) {
203 this.name = name;
204 }
205 public String getBgColor() {
206 return bgColor;
207 }
208 public void setBgColor(String bgColor) {
209 this.bgColor = bgColor;
210 }
211
212 LayoutArea(int width, int height) {
213 this.width = width;
214 this.height = height;
215 }
216
217 LayoutArea(String name, int x, int y, int width, int height, String bgColor) {
218 this(width, height);
219 this.name = name;
220 this.x = x;
221 this.y = y;
222 this.bgColor = bgColor;
223 }
224 }
225
226
227
228
229 class VideoInfo {
230 private int aspectRatioWidth = 16;
231 private int aspectRatioHeight = 9;
232
233 private long startTime = 0;
234 private long duration = 0;
235 private Track video;
236
237 public int getAspectRatioWidth() {
238 return aspectRatioWidth;
239 }
240 public void setAspectRatioWidth(int aspectRatioWidth) {
241 this.aspectRatioWidth = aspectRatioWidth;
242 }
243 public int getAspectRatioHeight() {
244 return aspectRatioHeight;
245 }
246 public void setAspectRatioHeight(int aspectRatioHeight) {
247 this.aspectRatioHeight = aspectRatioHeight;
248 }
249 public long getStartTime() {
250 return startTime;
251 }
252 public void setStartTime(long startTime) {
253 this.startTime = startTime;
254 }
255 public long getDuration() {
256 return duration;
257 }
258 public void setDuration(long duration) {
259 this.duration = duration;
260 }
261 public Track getVideo() {
262 return video;
263 }
264 public void setVideo(Track video) {
265 this.video = video;
266 }
267
268
269 VideoInfo() {
270
271 }
272
273 VideoInfo(int height, int width) {
274 aspectRatioWidth = width;
275 aspectRatioHeight = height;
276 }
277
278 VideoInfo(Track video, long timeStamp, int aspectRatioHeight, int aspectRatioWidth, long startTime) {
279 this(aspectRatioHeight, aspectRatioWidth);
280 this.video = video;
281 this.startTime = startTime;
282 }
283 }
284
285
286
287
288 class Offset {
289 private int x = 16;
290 private int y = 9;
291
292 public int getX() {
293 return x;
294 }
295 public void setX(int x) {
296 this.x = x;
297 }
298 public int getY() {
299 return y;
300 }
301 public void setY(int y) {
302 this.y = y;
303 }
304
305 Offset(int x, int y) {
306 this.x = x;
307 this.y = y;
308 }
309 }
310
311
312
313
314
315
316 class EditDecisionListSection {
317 private long timeStamp = 0;
318 private long nextTimeStamp = 0;
319 private List<VideoInfo> areas;
320
321 public long getTimeStamp() {
322 return timeStamp;
323 }
324 public void setTimeStamp(long timeStamp) {
325 this.timeStamp = timeStamp;
326 }
327 public long getNextTimeStamp() {
328 return nextTimeStamp;
329 }
330 public void setNextTimeStamp(long nextTimeStamp) {
331 this.nextTimeStamp = nextTimeStamp;
332 }
333 public List<VideoInfo> getAreas() {
334 return areas;
335 }
336 public void setAreas(List<VideoInfo> areas) {
337 this.areas = areas;
338 }
339
340 EditDecisionListSection() {
341 areas = new ArrayList<VideoInfo>();
342 }
343 }
344
345
346
347
348 class StartStopEvent implements Comparable<StartStopEvent> {
349 private boolean start;
350 private long timeStamp;
351 private Track video;
352 private VideoInfo videoInfo;
353
354 public boolean isStart() {
355 return start;
356 }
357 public void setStart(boolean start) {
358 this.start = start;
359 }
360 public long getTimeStamp() {
361 return timeStamp;
362 }
363 public void setTimeStamp(long timeStamp) {
364 this.timeStamp = timeStamp;
365 }
366 public VideoInfo getVideoInfo() {
367 return videoInfo;
368 }
369 public void setVideoInfo(VideoInfo videoInfo) {
370 this.videoInfo = videoInfo;
371 }
372
373 StartStopEvent(boolean start, Track video, long timeStamp, VideoInfo videoInfo) {
374 this.start = start;
375 this.timeStamp = timeStamp;
376 this.video = video;
377 this.videoInfo = videoInfo;
378 }
379
380 @Override
381 public int compareTo(StartStopEvent o) {
382 return Long.compare(this.timeStamp, o.timeStamp);
383 }
384 }
385
386 @Override
387 public WorkflowOperationResult start(final WorkflowInstance workflowInstance, JobContext context)
388 throws WorkflowOperationException {
389 logger.debug("Running videogrid workflow operation on workflow {}", workflowInstance.getId());
390
391 final MediaPackage mediaPackage = (MediaPackage) workflowInstance.getMediaPackage().clone();
392 ConfiguredTagsAndFlavors tagsAndFlavors = getTagsAndFlavors(workflowInstance,
393 Configuration.none, Configuration.many, Configuration.many, Configuration.one);
394
395
396 WorkflowOperationInstance operation = workflowInstance.getCurrentOperation();
397 final MediaPackageElementFlavor smilFlavor = MediaPackageElementFlavor.parseFlavor(
398 getConfig(operation, SOURCE_SMIL_FLAVOR));
399 final MediaPackageElementFlavor targetPresenterFlavor = tagsAndFlavors.getSingleTargetFlavor();
400 String concatEncodingProfile = StringUtils.trimToNull(operation.getConfiguration(CONCAT_ENCODING_PROFILE));
401
402
403 final List<MediaPackageElementFlavor> sourceFlavors = tagsAndFlavors.getSrcFlavors();
404
405
406 final List<Track> sourceTracks = new ArrayList<>();
407 for (MediaPackageElementFlavor sourceFlavor: sourceFlavors) {
408 TrackSelector trackSelector = new TrackSelector();
409 trackSelector.addFlavor(sourceFlavor);
410 sourceTracks.addAll(trackSelector.select(mediaPackage, false));
411 }
412
413
414 if (sourceTracks.isEmpty()) {
415 logger.warn("No tracks in source flavors, skipping ...");
416 return createResult(mediaPackage, WorkflowOperationResult.Action.SKIP);
417 }
418
419
420 if (concatEncodingProfile == null) {
421 throw new WorkflowOperationException("Encoding profile must be set!");
422 }
423 EncodingProfile profile = composerService.getProfile(concatEncodingProfile);
424 if (profile == null) {
425 throw new WorkflowOperationException("Encoding profile '" + concatEncodingProfile + "' was not found");
426 }
427
428
429
430 ImmutablePair<Integer, Integer> resolution;
431 try {
432 resolution = getResolution(getConfig(workflowInstance, OPT_RESOLUTION, "1280x720"));
433 } catch (IllegalArgumentException e) {
434 logger.warn("Given resolution was not well formatted!");
435 throw new WorkflowOperationException(e);
436 }
437 logger.info("The resolution of the final video: {}/{}", resolution.getLeft(), resolution.getRight());
438
439
440 String bgColor = getConfig(workflowInstance, OPT_BACKGROUND_COLOR, "0xFFFFFF");
441 final Pattern pattern = Pattern.compile("0x[A-Fa-f0-9]{6}");
442 if (!pattern.matcher(bgColor).matches()) {
443 logger.warn("Given color {} was not well formatted!", bgColor);
444 throw new WorkflowOperationException("Given color was not well formatted!");
445 }
446 logger.info("The background color of the final video: {}", bgColor);
447
448
449 List<String> targetTags = tagsAndFlavors.getTargetTags();
450
451
452 LayoutArea layoutArea = new LayoutArea("webcam", 0, 0, resolution.getLeft(), resolution.getRight(),
453 bgColor);
454
455
456 final SMILDocument smilDocument;
457 try {
458 smilDocument = SmilUtil.getSmilDocumentFromMediaPackage(mediaPackage, smilFlavor, workspace);
459 } catch (SAXException e) {
460 throw new WorkflowOperationException("SMIL is not well formatted", e);
461 } catch (IOException | NotFoundException e) {
462 throw new WorkflowOperationException("SMIL could not be found", e);
463 }
464
465 final SMILParElement parallel = (SMILParElement) smilDocument.getBody().getChildNodes().item(0);
466 final NodeList sequences = parallel.getTimeChildren();
467 final float trackDurationInSeconds = parallel.getDur();
468 final long trackDurationInMs = Math.round(trackDurationInSeconds * 1000f);
469
470
471 long finalStartTime = 0;
472 long finalEndTime = trackDurationInMs;
473
474
475
476 List<StartStopEvent> events = new ArrayList<>();
477 List<Track> videoSourceTracks = new ArrayList<>();
478
479 for (int i = 0; i < sequences.getLength(); i++) {
480 final SMILElement item = (SMILElement) sequences.item(i);
481 NodeList children = item.getChildNodes();
482
483 for (int j = 0; j < children.getLength(); j++) {
484 Node node = children.item(j);
485 SMILMediaElement e = (SMILMediaElement) node;
486
487
488 if (NODE_TYPE_VIDEO.equals(e.getNodeName())) {
489 Track track;
490 try {
491 track = getTrackByID(e.getId(), sourceTracks);
492 } catch (IllegalStateException ex) {
493 logger.info("No track corresponding to SMIL ID found, skipping SMIL ID {}", e.getId());
494 continue;
495 }
496 videoSourceTracks.add(track);
497
498 double beginInSeconds = e.getBegin().item(0).getResolvedOffset();
499 long beginInMs = Math.round(beginInSeconds * 1000d);
500 double durationInSeconds = e.getDur();
501 long durationInMs = Math.round(durationInSeconds * 1000d);
502
503
504 VideoInfo videoInfo = new VideoInfo();
505
506 List<Track> tmpList = new ArrayList<Track>();
507 tmpList.add(track);
508 LayoutArea trackDimension = determineDimension(tmpList, true);
509 if (trackDimension == null) {
510 throw new WorkflowOperationException("One of the source video tracks did not contain "
511 + "a valid video stream or dimension");
512 }
513 videoInfo.aspectRatioHeight = trackDimension.getHeight();
514 videoInfo.aspectRatioWidth = trackDimension.getWidth();
515
516
517 videoInfo.startTime = 0;
518
519 logger.info("Video information: Width: {}, Height {}, StartTime: {}", videoInfo.aspectRatioWidth,
520 videoInfo.aspectRatioHeight, videoInfo.startTime);
521
522 events.add(new StartStopEvent(true, track, beginInMs, videoInfo));
523 events.add(new StartStopEvent(false, track, beginInMs + durationInMs, videoInfo));
524
525 }
526 }
527 }
528
529
530 if (events.isEmpty()) {
531 logger.warn("Could not generate sections from given SMIL catalogue for tracks in given flavor, skipping ...");
532 return createResult(mediaPackage, WorkflowOperationResult.Action.SKIP);
533 }
534
535
536 Collections.sort(events);
537
538
539 List<EditDecisionListSection> videoEdl = new ArrayList<EditDecisionListSection>();
540 HashMap<Track, StartStopEvent> activeVideos = new HashMap<>();
541
542
543 EditDecisionListSection start = new EditDecisionListSection();
544 start.timeStamp = finalStartTime;
545 videoEdl.add(start);
546
547
548 for (StartStopEvent event : events) {
549 if (event.start) {
550 logger.info("Add start event at {}", event.timeStamp);
551 activeVideos.put(event.video, event);
552 } else {
553 logger.info("Add stop event at {}", event);
554 activeVideos.remove(event.video);
555 }
556 videoEdl.add(createEditDecisionList(event, activeVideos));
557 }
558
559
560 EditDecisionListSection endVideo = new EditDecisionListSection();
561 endVideo.timeStamp = finalEndTime;
562 endVideo.nextTimeStamp = finalEndTime;
563 videoEdl.add(endVideo);
564
565
566 for (int i = 0; i < videoEdl.size() - 1; i++) {
567
568 videoEdl.get(i).nextTimeStamp = videoEdl.get(i + 1).timeStamp;
569 }
570
571
572 List<List<String>> commands = new ArrayList<>();
573 List<List<Track>> tracksForCommands = new ArrayList<>();
574 for (EditDecisionListSection edl : videoEdl) {
575
576 if (edl.nextTimeStamp - edl.timeStamp < 50) {
577 logger.info("Skipping {}-length edl entry", edl.nextTimeStamp - edl.timeStamp);
578 continue;
579 }
580
581 commands.add(compositeSection(layoutArea, edl));
582 tracksForCommands.add(edl.getAreas().stream().map(m -> m.getVideo()).collect(Collectors.toList()));
583 }
584
585
586 List<URI> uris = new ArrayList<>();
587 for (int i = 0; i < commands.size(); i++) {
588 logger.info("Sending command {} of {} to service. Command: {}", i + 1, commands.size(), commands.get(i));
589
590 Job job;
591 try {
592 job = videoGridService.createPartialTrack(
593 commands.get(i),
594 tracksForCommands.get(i).toArray(new Track[tracksForCommands.get(i).size()])
595 );
596 } catch (VideoGridServiceException | org.apache.commons.codec.EncoderException | MediaPackageException e) {
597 throw new WorkflowOperationException(e);
598 }
599
600 if (!waitForStatus(job).isSuccess()) {
601 throw new WorkflowOperationException(
602 String.format("VideoGrid job for media package '%s' failed", mediaPackage));
603 }
604
605 Gson gson = new Gson();
606 uris.add(gson.fromJson(job.getPayload(), new TypeToken<URI>() { }.getType()));
607 }
608
609
610 List<Track> tracks = new ArrayList<>();
611 for (URI uri : uris) {
612 TrackImpl track = new TrackImpl();
613 track.setFlavor(targetPresenterFlavor);
614 track.setURI(uri);
615
616 Job inspection = null;
617 try {
618 inspection = inspectionService.enrich(track, true);
619 } catch (MediaInspectionException | MediaPackageException e) {
620 throw new WorkflowOperationException("Inspection service could not enrich track", e);
621 }
622 if (!waitForStatus(inspection).isSuccess()) {
623 throw new WorkflowOperationException(String.format("Failed to add metadata to track."));
624 }
625
626 try {
627 tracks.add((TrackImpl) MediaPackageElementParser.getFromXml(inspection.getPayload()));
628 } catch (MediaPackageException e) {
629 throw new WorkflowOperationException("Could not parse track returned by inspection service", e);
630 }
631 }
632
633
634 Job concatJob = null;
635 try {
636 concatJob = composerService.concat(composerService.getProfile(concatEncodingProfile).getIdentifier(),
637 new Dimension(layoutArea.width,layoutArea.height) , true, tracks.toArray(new Track[tracks.size()]));
638 } catch (EncoderException | MediaPackageException e) {
639 throw new WorkflowOperationException("The concat job failed", e);
640 }
641 if (!waitForStatus(concatJob).isSuccess()) {
642 throw new WorkflowOperationException("The concat job did not complete successfully.");
643 }
644
645
646 if (concatJob.getPayload().length() > 0) {
647 Track concatTrack;
648 try {
649 concatTrack = (Track) MediaPackageElementParser.getFromXml(concatJob.getPayload());
650 } catch (MediaPackageException e) {
651 throw new WorkflowOperationException("Could not parse track returned by concat service", e);
652 }
653 concatTrack.setFlavor(targetPresenterFlavor);
654 concatTrack.setURI(concatTrack.getURI());
655 for (String tag : targetTags) {
656 concatTrack.addTag(tag);
657 }
658
659 mediaPackage.add(concatTrack);
660 } else {
661 throw new WorkflowOperationException("Concat operation unsuccessful, no payload returned.");
662 }
663
664 try {
665 workspace.cleanup(mediaPackage.getIdentifier());
666 } catch (IOException e) {
667 throw new WorkflowOperationException(e);
668 }
669
670 final WorkflowOperationResult result = createResult(mediaPackage, WorkflowOperationResult.Action.CONTINUE);
671 logger.debug("Video Grid operation completed");
672 return result;
673 }
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691 private List<String> compositeSection(LayoutArea layoutArea, EditDecisionListSection videoEdl) {
692
693 long duration = videoEdl.nextTimeStamp - videoEdl.timeStamp;
694 logger.info("Cut timeStamp {}, duration {}", videoEdl.timeStamp, duration);
695
696
697 String ffmpegFilter = String.format("color=c=%s:s=%dx%d:r=24", layoutArea.bgColor,
698 layoutArea.width, layoutArea.height);
699
700 List<VideoInfo> videos = videoEdl.areas;
701 int videoCount = videoEdl.areas.size();
702
703 logger.info("Laying out {} videos in {}", videoCount, layoutArea.name);
704
705
706 if (videoCount > 0) {
707 int tilesH = 0;
708 int tilesV = 0;
709 int tileWidth = 0;
710 int tileHeight = 0;
711 int totalArea = 0;
712
713
714 for (int tmpTilesV = 1; tmpTilesV < videoCount + 1; tmpTilesV++) {
715 int tmpTilesH = (int) Math.ceil((videoCount / (float)tmpTilesV));
716 int tmpTileWidth = (int) (2 * Math.floor((float)layoutArea.width / tmpTilesH / 2));
717 int tmpTileHeight = (int) (2 * Math.floor((float)layoutArea.height / tmpTilesV / 2));
718
719 if (tmpTileWidth <= 0 || tmpTileHeight <= 0) {
720 continue;
721 }
722
723 int tmpTotalArea = 0;
724 for (VideoInfo video: videos) {
725 int videoWidth = video.aspectRatioWidth;
726 int videoHeight = video.aspectRatioHeight;
727 VideoInfo videoScaled = aspectScale(videoWidth, videoHeight, tmpTileWidth, tmpTileHeight);
728 tmpTotalArea += videoScaled.aspectRatioWidth * videoScaled.aspectRatioHeight;
729 }
730
731 if (tmpTotalArea > totalArea) {
732 tilesH = tmpTilesH;
733 tilesV = tmpTilesV;
734 tileWidth = tmpTileWidth;
735 tileHeight = tmpTileHeight;
736 totalArea = tmpTotalArea;
737 }
738 }
739
740
741 int tileX = 0;
742 int tileY = 0;
743
744 logger.info("Tiling in a {}x{} grid", tilesH, tilesV);
745
746 ffmpegFilter += String.format("[%s_in];", layoutArea.name);
747
748 for (VideoInfo video : videos) {
749
750 logger.info("tile location ({}, {})", tileX, tileY);
751 int videoWidth = video.aspectRatioWidth;
752 int videoHeight = video.aspectRatioHeight;
753 logger.info("original aspect: {}x{}", videoWidth, videoHeight);
754
755 VideoInfo videoScaled = aspectScale(videoWidth, videoHeight, tileWidth, tileHeight);
756 logger.info("scaled size: {}x{}", videoScaled.aspectRatioWidth, videoScaled.aspectRatioHeight);
757
758 Offset offset = padOffset(videoScaled.aspectRatioWidth, videoScaled.aspectRatioHeight, tileWidth, tileHeight);
759 logger.info("offset: left: {}, top: {}", offset.x, offset.y);
760
761
762
763 long seekOffset = 0;
764 logger.info("seek offset: {}", seekOffset);
765
766
767
768
769 long seek = video.startTime - 10000;
770 if (seek < 0) {
771 seek = 0;
772 }
773
774 String padName = String.format("%s_x%d_y%d", layoutArea.name, tileX, tileY);
775
776
777
778
779
780 if (seek > 0) {
781 seek = seek + seekOffset;
782 }
783
784
785 ffmpegFilter += String.format("movie=%s:sp=%s", "#{" + video.getVideo().getIdentifier() + "}", msToS(seek));
786
787
788 ffmpegFilter += String.format(",setpts=PTS-%s/TB", msToS(seekOffset));
789
790
791 ffmpegFilter += String.format(",fps=%d:start_time=%s", FFMPEG_WF_FRAMERATE, msToS(video.startTime));
792
793
794 ffmpegFilter += String.format(",setpts=PTS-STARTPTS,scale=%d:%d,setsar=1",
795 videoScaled.aspectRatioWidth, videoScaled.aspectRatioHeight);
796
797 ffmpegFilter += String.format(",pad=w=%d:h=%d:x=%d:y=%d:color=%s", tileWidth, tileHeight,
798 offset.x, offset.y, layoutArea.bgColor);
799 ffmpegFilter += String.format("[%s_movie];", padName);
800
801
802
803
804
805 ffmpegFilter += String.format("color=c=%s:s=%dx%d:r=%d", layoutArea.bgColor, tileWidth,
806 tileHeight, FFMPEG_WF_FRAMERATE);
807 ffmpegFilter += String.format("[%s_pad];", padName);
808 ffmpegFilter += String.format("[%s_movie][%s_pad]concat=n=2:v=1:a=0[%s];", padName, padName, padName);
809
810 tileX += 1;
811 if (tileX >= tilesH) {
812 tileX = 0;
813 tileY += 1;
814 }
815 }
816
817
818 int remaining = videoCount;
819 for (tileY = 0; tileY < tilesV; tileY++) {
820 int thisTilesH = Math.min(tilesH, remaining);
821 remaining -= thisTilesH;
822
823 for (tileX = 0; tileX < thisTilesH; tileX++) {
824 ffmpegFilter += String.format("[%s_x%d_y%d]", layoutArea.name, tileX, tileY);
825 }
826 if (thisTilesH > 1) {
827 ffmpegFilter += String.format("hstack=inputs=%d,", thisTilesH);
828 }
829 ffmpegFilter += String.format("pad=w=%d:h=%d:color=%s", layoutArea.width, tileHeight, layoutArea.bgColor);
830 ffmpegFilter += String.format("[%s_y%d];", layoutArea.name, tileY);
831 }
832
833
834 for (tileY = 0; tileY < tilesV; tileY++) {
835 ffmpegFilter += String.format("[%s_y%d]", layoutArea.name, tileY);
836 }
837 if (tilesV > 1) {
838 ffmpegFilter += String.format("vstack=inputs=%d,", tilesV);
839 }
840 ffmpegFilter += String.format("pad=w=%d:h=%d:color=%s", layoutArea.width, layoutArea.height, layoutArea.bgColor);
841 ffmpegFilter += String.format("[%s];", layoutArea.name);
842 ffmpegFilter += String.format("[%s_in][%s]overlay=x=%d:y=%d", layoutArea.name,
843 layoutArea.name, layoutArea.x, layoutArea.y);
844
845
846 }
847
848 ffmpegFilter += String.format(",trim=end=%s", msToS(duration));
849
850 List<String> ffmpegCmd = new ArrayList<String>(Arrays.asList(FFMPEG));
851 ffmpegCmd.add("-filter_complex");
852 ffmpegCmd.add(ffmpegFilter);
853 ffmpegCmd.addAll(Arrays.asList(FFMPEG_WF_ARGS));
854
855 logger.info("Final command:");
856 logger.info(String.join(" ", ffmpegCmd));
857
858 return ffmpegCmd;
859 }
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874 private VideoInfo aspectScale(int oldWidth, int oldHeight, int newWidth, int newHeight) {
875 if ((float)oldWidth / oldHeight > (float)newWidth / newHeight) {
876 newHeight = (int) (2 * Math.round((float)oldHeight * newWidth / oldWidth / 2));
877 } else {
878 newWidth = (int) (2 * Math.round((float)oldWidth * newHeight / oldHeight / 2));
879 }
880 return new VideoInfo(newHeight, newWidth);
881 }
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896 private Offset padOffset(int videoWidth, int videoHeight, int areaWidth, int areaHeight) {
897 int padX = (int) (2 * Math.round((float)(areaWidth - videoWidth) / 4));
898 int padY = (int) (2 * Math.round((float)(areaHeight - videoHeight) / 4));
899 return new Offset(padX, padY);
900 }
901
902
903
904
905
906
907
908
909 private String msToS(long timestamp) {
910 double s = (double)timestamp / 1000;
911 return String.format(Locale.US, "%.3f", s);
912 }
913
914
915
916
917
918
919
920
921
922
923 private Track getTrackByID(String trackId, List<Track> tracks) {
924 for (Track t : tracks) {
925 if (t.getIdentifier().contains(trackId)) {
926 logger.debug("Track-Id from smil found in Mediapackage ID: " + t.getIdentifier());
927 return t;
928 }
929 }
930 throw new IllegalStateException("No track matching smil Track-id: " + trackId);
931 }
932
933
934
935
936
937
938
939
940
941
942 private LayoutArea determineDimension(List<Track> tracks, boolean forceDivisible) {
943 Tuple<Track, LayoutArea> trackDimension = getLargestTrack(tracks);
944 if (trackDimension == null) {
945 return null;
946 }
947
948 if (forceDivisible && (trackDimension.getB().getHeight() % 2 != 0 || trackDimension.getB().getWidth() % 2 != 0)) {
949 LayoutArea scaledDimension = new LayoutArea((trackDimension.getB().getWidth() / 2) * 2, (trackDimension
950 .getB().getHeight() / 2) * 2);
951 logger.info("Determined output dimension {} scaled down from {} for track {}", scaledDimension,
952 trackDimension.getB(), trackDimension.getA());
953 return scaledDimension;
954 } else {
955 logger.info("Determined output dimension {} for track {}", trackDimension.getB(), trackDimension.getA());
956 return trackDimension.getB();
957 }
958 }
959
960
961
962
963
964
965
966
967 private Tuple<Track, LayoutArea> getLargestTrack(List<Track> tracks) {
968 Track track = null;
969 LayoutArea dimension = null;
970 for (Track t : tracks) {
971 if (!t.hasVideo()) {
972 continue;
973 }
974
975 VideoStream[] videoStreams = TrackSupport.byType(t.getStreams(), VideoStream.class);
976 int frameWidth = videoStreams[0].getFrameWidth();
977 int frameHeight = videoStreams[0].getFrameHeight();
978 if (dimension == null || (frameWidth * frameHeight) > (dimension.getWidth() * dimension.getHeight())) {
979 dimension = new LayoutArea(frameWidth, frameHeight);
980 track = t;
981 }
982 }
983 if (track == null || dimension == null) {
984 return null;
985 }
986
987 return Tuple.tuple(track, dimension);
988 }
989
990
991
992
993
994
995
996
997
998 private String getTrackPath(Track track) throws WorkflowOperationException {
999 File mediaFile;
1000 try {
1001 mediaFile = workspace.get(track.getURI());
1002 } catch (NotFoundException e) {
1003 throw new WorkflowOperationException(
1004 "Error finding the media file in the workspace", e);
1005 } catch (IOException e) {
1006 throw new WorkflowOperationException(
1007 "Error reading the media file in the workspace", e);
1008 }
1009 return mediaFile.getAbsolutePath();
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 private EditDecisionListSection createEditDecisionList(
1021 StartStopEvent event,
1022 HashMap<Track, StartStopEvent> activeVideos
1023 ) {
1024 EditDecisionListSection nextEdl = new EditDecisionListSection();
1025 nextEdl.timeStamp = event.timeStamp;
1026
1027 for (Map.Entry<Track, StartStopEvent> activeVideo : activeVideos.entrySet()) {
1028 nextEdl.areas.add(new VideoInfo(activeVideo.getKey(), event.timeStamp,
1029 activeVideo.getValue().videoInfo.aspectRatioHeight,
1030 activeVideo.getValue().videoInfo.aspectRatioWidth,
1031 event.timeStamp - activeVideo.getValue().timeStamp));
1032 }
1033
1034 return nextEdl;
1035 }
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045 private ImmutablePair<Integer, Integer> getResolution(String s) throws IllegalArgumentException {
1046 String[] parts = s.split("x");
1047 if (parts.length != 2) {
1048 throw new IllegalArgumentException(format("Unable to create resolution from \"%s\"", s));
1049 }
1050
1051 return new ImmutablePair<Integer, Integer>(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]));
1052 }
1053 }