1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.videosegmenter.ffmpeg;
23
24 import org.opencastproject.job.api.AbstractJobProducer;
25 import org.opencastproject.job.api.Job;
26 import org.opencastproject.mediapackage.Catalog;
27 import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
28 import org.opencastproject.mediapackage.MediaPackageElementParser;
29 import org.opencastproject.mediapackage.MediaPackageElements;
30 import org.opencastproject.mediapackage.MediaPackageException;
31 import org.opencastproject.mediapackage.Track;
32 import org.opencastproject.metadata.mpeg7.MediaLocator;
33 import org.opencastproject.metadata.mpeg7.MediaLocatorImpl;
34 import org.opencastproject.metadata.mpeg7.MediaRelTimeImpl;
35 import org.opencastproject.metadata.mpeg7.MediaTime;
36 import org.opencastproject.metadata.mpeg7.MediaTimePoint;
37 import org.opencastproject.metadata.mpeg7.MediaTimePointImpl;
38 import org.opencastproject.metadata.mpeg7.Mpeg7Catalog;
39 import org.opencastproject.metadata.mpeg7.Mpeg7CatalogService;
40 import org.opencastproject.metadata.mpeg7.Segment;
41 import org.opencastproject.metadata.mpeg7.Video;
42 import org.opencastproject.security.api.OrganizationDirectoryService;
43 import org.opencastproject.security.api.SecurityService;
44 import org.opencastproject.security.api.UserDirectoryService;
45 import org.opencastproject.serviceregistry.api.ServiceRegistry;
46 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
47 import org.opencastproject.util.LoadUtil;
48 import org.opencastproject.util.MimeType;
49 import org.opencastproject.util.MimeTypes;
50 import org.opencastproject.util.NotFoundException;
51 import org.opencastproject.videosegmenter.api.VideoSegmenterException;
52 import org.opencastproject.videosegmenter.api.VideoSegmenterService;
53 import org.opencastproject.workspace.api.Workspace;
54
55 import org.apache.commons.lang3.BooleanUtils;
56 import org.apache.commons.lang3.StringUtils;
57 import org.osgi.service.cm.ConfigurationException;
58 import org.osgi.service.cm.ManagedService;
59 import org.osgi.service.component.ComponentContext;
60 import org.osgi.service.component.annotations.Component;
61 import org.osgi.service.component.annotations.Reference;
62 import org.slf4j.Logger;
63 import org.slf4j.LoggerFactory;
64
65 import java.io.BufferedReader;
66 import java.io.File;
67 import java.io.IOException;
68 import java.io.InputStreamReader;
69 import java.net.URI;
70 import java.net.URL;
71 import java.text.ParseException;
72 import java.util.ArrayList;
73 import java.util.Arrays;
74 import java.util.Collections;
75 import java.util.Dictionary;
76 import java.util.LinkedList;
77 import java.util.List;
78 import java.util.Optional;
79 import java.util.regex.Matcher;
80 import java.util.regex.Pattern;
81 import java.util.stream.Collectors;
82
83
84
85
86
87
88
89
90
91
92
93 @Component(
94 immediate = true,
95 service = { VideoSegmenterService.class,ManagedService.class },
96 property = {
97 "service.description=VideoSegmenter Service"
98 }
99 )
100 public class VideoSegmenterServiceImpl extends AbstractJobProducer implements
101 VideoSegmenterService, ManagedService {
102
103
104 public static final String COLLECTION_ID = "videosegments";
105
106
107 private enum Operation {
108 Segment
109 };
110
111 private class Chapter {
112 protected double start;
113 protected double end;
114 protected Optional<String> title;
115 };
116
117
118 protected String binary;
119
120 public static final String FFMPEG_BINARY_CONFIG = "org.opencastproject.composer.ffmpeg.path";
121 public static final String FFMPEG_BINARY_DEFAULT = "ffmpeg";
122
123
124 public static final String OPT_STABILITY_THRESHOLD = "stabilitythreshold";
125
126
127 public static final int DEFAULT_STABILITY_THRESHOLD = 60;
128
129
130 public static final String OPT_CHANGES_THRESHOLD = "changesthreshold";
131
132
133 public static final float DEFAULT_CHANGES_THRESHOLD = 0.025f;
134
135
136 public static final String OPT_PREF_NUMBER = "prefNumber";
137
138
139 public static final int DEFAULT_PREF_NUMBER = 30;
140
141
142 public static final String OPT_MAX_CYCLES = "maxCycles";
143
144
145 public static final int DEFAULT_MAX_CYCLES = 3;
146
147
148 public static final String OPT_MAX_ERROR = "maxError";
149
150
151 public static final float DEFAULT_MAX_ERROR = 0.25f;
152
153
154 public static final String OPT_ABSOLUTE_MAX = "absoluteMax";
155
156
157 public static final int DEFAULT_ABSOLUTE_MAX = 150;
158
159
160 public static final String OPT_ABSOLUTE_MIN = "absoluteMin";
161
162
163 public static final int DEFAULT_ABSOLUTE_MIN = 3;
164
165
166 public static final String OPT_DURATION_DEPENDENT = "durationDependent";
167
168
169 public static final boolean DEFAULT_DURATION_DEPENDENT = false;
170
171
172 public static final String OPT_USE_CHAPTER_IF_AVAILABLE = "useChapterIfAvailable";
173
174
175 public static final boolean DEFAULT_USE_CHAPTER_IF_AVAILABLE = false;
176
177 private boolean useChapterIfAvailable = DEFAULT_USE_CHAPTER_IF_AVAILABLE;
178
179
180 public static final String OPT_USE_CHAPTER_MIME_TYPES = "useChapterMimeTypes";
181
182 public static final List<MimeType> DEFAULT_USE_CHAPTER_MIME_TYPES = new ArrayList<>();
183
184 private List<MimeType> useChapterMimeTypes = DEFAULT_USE_CHAPTER_MIME_TYPES;
185
186
187 public static final float DEFAULT_SEGMENTER_JOB_LOAD = 0.3f;
188
189
190 public static final String SEGMENTER_JOB_LOAD_KEY = "job.load.videosegmenter";
191
192
193 private float segmenterJobLoad = DEFAULT_SEGMENTER_JOB_LOAD;
194
195
196 protected static final Logger logger = LoggerFactory
197 .getLogger(VideoSegmenterServiceImpl.class);
198
199
200 protected float changesThreshold = DEFAULT_CHANGES_THRESHOLD;
201
202
203 protected int stabilityThreshold = DEFAULT_STABILITY_THRESHOLD;
204
205
206 protected int stabilityThresholdPrefilter = 1;
207
208
209 protected int prefNumber = DEFAULT_PREF_NUMBER;
210
211
212 protected int maxCycles = DEFAULT_MAX_CYCLES;
213
214
215 protected float maxError = DEFAULT_MAX_ERROR;
216
217
218 protected int absoluteMax = DEFAULT_ABSOLUTE_MAX;
219
220
221 protected int absoluteMin = DEFAULT_ABSOLUTE_MIN;
222
223
224 protected boolean durationDependent = DEFAULT_DURATION_DEPENDENT;
225
226
227 protected ServiceRegistry serviceRegistry = null;
228
229
230 protected Mpeg7CatalogService mpeg7CatalogService = null;
231
232
233 protected Workspace workspace = null;
234
235
236 protected SecurityService securityService = null;
237
238
239 protected UserDirectoryService userDirectoryService = null;
240
241
242 protected OrganizationDirectoryService organizationDirectoryService = null;
243
244
245
246
247 public VideoSegmenterServiceImpl() {
248 super(JOB_TYPE);
249 this.binary = FFMPEG_BINARY_DEFAULT;
250 }
251
252 @Override
253 public void activate(ComponentContext cc) {
254 super.activate(cc);
255
256 final String path = cc.getBundleContext().getProperty(FFMPEG_BINARY_CONFIG);
257 this.binary = path == null ? FFMPEG_BINARY_DEFAULT : path;
258 logger.debug("Configuration {}: {}", FFMPEG_BINARY_CONFIG, FFMPEG_BINARY_DEFAULT);
259 }
260
261
262
263
264
265
266 @Override
267 public void updated(Dictionary<String, ?> properties) throws ConfigurationException {
268 if (properties == null) {
269 return;
270 }
271 logger.debug("Configuring the videosegmenter");
272
273
274 if (properties.get(OPT_STABILITY_THRESHOLD) != null) {
275 String threshold = (String) properties.get(OPT_STABILITY_THRESHOLD);
276 try {
277 stabilityThreshold = Integer.parseInt(threshold);
278 logger.info("Stability threshold set to {} consecutive frames", stabilityThreshold);
279 } catch (Exception e) {
280 throw new ConfigurationException(OPT_STABILITY_THRESHOLD,
281 String.format("Found illegal value '%s'", threshold)
282 );
283 }
284 }
285
286
287 if (properties.get(OPT_CHANGES_THRESHOLD) != null) {
288 String threshold = (String) properties.get(OPT_CHANGES_THRESHOLD);
289 try {
290 changesThreshold = Float.parseFloat(threshold);
291 logger.info("Changes threshold set to {}", changesThreshold);
292 } catch (Exception e) {
293 throw new ConfigurationException(OPT_CHANGES_THRESHOLD,
294 String.format("Found illegal value '%s'", threshold)
295 );
296 }
297 }
298
299
300 if (properties.get(OPT_PREF_NUMBER) != null) {
301 String number = (String) properties.get(OPT_PREF_NUMBER);
302 try {
303 prefNumber = Integer.parseInt(number);
304 logger.info("Preferred number of segments set to {}", prefNumber);
305 } catch (Exception e) {
306 throw new ConfigurationException(OPT_PREF_NUMBER,
307 String.format("Found illegal value '%s'", number)
308 );
309 }
310 }
311
312
313 if (properties.get(OPT_MAX_CYCLES) != null) {
314 String number = (String) properties.get(OPT_MAX_CYCLES);
315 try {
316 maxCycles = Integer.parseInt(number);
317 logger.info("Maximum number of cycles set to {}", maxCycles);
318 } catch (Exception e) {
319 throw new ConfigurationException(OPT_MAX_CYCLES,
320 String.format("Found illegal value '%s'", number)
321 );
322 }
323 }
324
325
326 if (properties.get(OPT_ABSOLUTE_MAX) != null) {
327 String number = (String) properties.get(OPT_ABSOLUTE_MAX);
328 try {
329 absoluteMax = Integer.parseInt(number);
330 logger.info("Absolute maximum number of segments set to {}", absoluteMax);
331 } catch (Exception e) {
332 throw new ConfigurationException(OPT_ABSOLUTE_MAX,
333 String.format("Found illegal value '%s'", number)
334 );
335 }
336 }
337
338
339 if (properties.get(OPT_ABSOLUTE_MIN) != null) {
340 String number = (String) properties.get(OPT_ABSOLUTE_MIN);
341 try {
342 absoluteMin = Integer.parseInt(number);
343 logger.info("Absolute minimum number of segments set to {}", absoluteMin);
344 } catch (Exception e) {
345 throw new ConfigurationException(OPT_ABSOLUTE_MIN,
346 String.format("Found illegal value '%s'", number)
347 );
348 }
349 }
350
351
352 if (properties.get(OPT_DURATION_DEPENDENT) != null) {
353 String value = (String) properties.get(OPT_DURATION_DEPENDENT);
354 try {
355 durationDependent = BooleanUtils.toBooleanObject(StringUtils.trimToNull(value));
356 logger.info("Dependency on video duration is set to {}", durationDependent);
357 } catch (Exception e) {
358 throw new ConfigurationException(OPT_DURATION_DEPENDENT,
359 String.format("Found illegal value '%s'", value)
360 );
361 }
362 }
363
364 if (properties.get(OPT_USE_CHAPTER_IF_AVAILABLE) != null) {
365 String value = (String) properties.get(OPT_USE_CHAPTER_IF_AVAILABLE);
366 try {
367 useChapterIfAvailable = BooleanUtils.toBooleanObject(StringUtils.trimToNull(value));
368 logger.info("Use Chapters if available is set to {}", useChapterIfAvailable);
369 } catch (Exception e) {
370 throw new ConfigurationException(OPT_USE_CHAPTER_IF_AVAILABLE,
371 String.format("Found illegal value '%s'", value)
372 );
373 }
374 }
375
376 if (properties.get(OPT_USE_CHAPTER_MIME_TYPES) != null) {
377 String value = (String) properties.get(OPT_USE_CHAPTER_MIME_TYPES);
378 try {
379 List<MimeType> mts = new ArrayList<>();
380 String[] values = value.split(",");
381
382 for (String mimeString : values) {
383 MimeType mt = MimeTypes.parseMimeType(mimeString);
384 mts.add(mt);
385 }
386
387 useChapterMimeTypes = mts;
388 } catch (Exception e) {
389 throw new ConfigurationException(OPT_USE_CHAPTER_MIME_TYPES,
390 String.format("Found illegal value '%s'", value)
391 );
392 }
393 } else {
394 useChapterMimeTypes = DEFAULT_USE_CHAPTER_MIME_TYPES;
395 }
396
397 segmenterJobLoad = LoadUtil.getConfiguredLoadValue(
398 properties, SEGMENTER_JOB_LOAD_KEY, DEFAULT_SEGMENTER_JOB_LOAD, serviceRegistry);
399 }
400
401
402
403
404
405
406 public Job segment(Track track) throws VideoSegmenterException,
407 MediaPackageException {
408 try {
409 return serviceRegistry.createJob(JOB_TYPE,
410 Operation.Segment.toString(),
411 Arrays.asList(MediaPackageElementParser.getAsXml(track)), segmenterJobLoad);
412 } catch (ServiceRegistryException e) {
413 throw new VideoSegmenterException("Unable to create a job", e);
414 }
415 }
416
417
418
419
420
421
422
423
424
425
426
427 protected Catalog segment(Job job, Track track)
428 throws VideoSegmenterException, MediaPackageException {
429
430
431
432 if (!track.hasVideo()) {
433 logger.warn("Element {} is not a video track", track);
434 throw new VideoSegmenterException("Element is not a video track");
435 }
436
437 try {
438 File mediaFile = null;
439 URL mediaUrl = null;
440 try {
441 mediaFile = workspace.get(track.getURI());
442 mediaUrl = mediaFile.toURI().toURL();
443 } catch (NotFoundException e) {
444 throw new VideoSegmenterException(
445 "Error finding the video file in the workspace", e);
446 } catch (IOException e) {
447 throw new VideoSegmenterException(
448 "Error reading the video file in the workspace", e);
449 }
450
451 if (track.getDuration() == null) {
452 throw new MediaPackageException("Track " + track
453 + " does not have a duration");
454 }
455 logger.info("Track {} loaded, duration is {} s", mediaUrl,
456 track.getDuration() / 1000);
457
458 Mpeg7Catalog mpeg7;
459 Optional<List<Chapter>> chapter = Optional.empty();
460 if (useChapterIfAvailable
461 && (useChapterMimeTypes.isEmpty()
462 || useChapterMimeTypes.stream().anyMatch(comp -> track.getMimeType().eq(comp)))) {
463 chapter = Optional.ofNullable(extractChapter(mediaFile));
464 }
465 if (chapter.isPresent() && !chapter.get().isEmpty()) {
466 mpeg7 = segmentFromChapter(chapter.get(), track);
467 } else {
468 mpeg7 = segmentAndOptimize(track, mediaFile, mediaUrl);
469 }
470
471 Catalog mpeg7Catalog = (Catalog) MediaPackageElementBuilderFactory
472 .newInstance().newElementBuilder()
473 .newElement(Catalog.TYPE, MediaPackageElements.SEGMENTS);
474 URI uri;
475 try {
476 uri = workspace.putInCollection(COLLECTION_ID, job.getId()
477 + ".xml", mpeg7CatalogService.serialize(mpeg7));
478 } catch (IOException e) {
479 throw new VideoSegmenterException(
480 "Unable to put the mpeg7 catalog into the workspace", e);
481 }
482 mpeg7Catalog.setURI(uri);
483
484 logger.info("Finished video segmentation of {}", mediaUrl);
485 return mpeg7Catalog;
486 } catch (Exception e) {
487 logger.warn("Error segmenting " + track, e);
488 if (e instanceof VideoSegmenterException) {
489 throw (VideoSegmenterException) e;
490 } else {
491 throw new VideoSegmenterException(e);
492 }
493 }
494 }
495
496
497
498
499
500
501 private List<Chapter> extractChapter(final File mediaFile) throws IOException {
502 String[] command = new String[] {
503 binary,
504 "-nostats", "-nostdin",
505 "-i", mediaFile.getAbsolutePath(),
506 "-f", "FFMETADATA",
507 "-"
508 };
509
510 logger.debug("Detecting chapters using command: {}", (Object) command);
511
512 ProcessBuilder pbuilder = new ProcessBuilder(command);
513 Process process = pbuilder.start();
514 try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
515 return parseChapter(reader);
516 } catch (IOException e) {
517 logger.error("Error executing ffmpeg: {}", e.getMessage());
518 } catch (ParseException e) {
519 logger.error("Error parsing ffmpeg output: {}", e.getMessage());
520 }
521
522 return null;
523 }
524
525
526
527
528
529
530
531
532 private List<Chapter> parseChapter(final BufferedReader reader) throws IOException, ParseException {
533 List<Chapter> chapters = new ArrayList<Chapter>();
534
535 int state = 0;
536
537 final double defaultTimebase = 1e-9f;
538 double timebase = defaultTimebase;
539 long start = -1;
540 long end = -1;
541 Optional<StringBuilder> title = Optional.empty();
542
543 String line = reader.readLine();
544 int lineNumber = 1;
545 if (line == null) {
546 return chapters;
547 }
548 while (true) {
549
550 if (state == 0 && ";FFMETADATA1".equals(line)) {
551 state++;
552 }
553
554 else if (line != null && (line.startsWith(";") || line.startsWith("#") || line.isEmpty())) { }
555
556 else if (state == 1 && "[CHAPTER]".equals(line)) {
557 state++;
558 }
559
560 else if (state == 2) {
561
562 if (!line.startsWith("TIMEBASE=")) {
563 state++;
564 continue;
565 }
566
567 String[] timebaseSplit = line.split("=");
568
569 if (timebaseSplit.length != 2) {
570 throw new ParseException("Failed to parse FFMETADATA:"
571 + " CHAPTER TIMEBASE line not correctly formatted", lineNumber);
572 }
573
574 String ratio = timebaseSplit[1];
575 String[] numbers = ratio.split("/");
576
577 if (numbers.length != 2) {
578 throw new ParseException("Failed to parse FFMETADATA: ratio not correctly formatted", lineNumber);
579 }
580
581 try {
582
583 timebase = Double.parseDouble(numbers[0]) / Double.parseDouble(numbers[1]);
584 }
585 catch (NumberFormatException e) {
586 throw new ParseException("Failed to parse FFMETADATA:"
587 + " Couldn't parse timebase as ratio of integer numbers", lineNumber);
588 }
589
590 state++;
591 }
592
593 else if (state == 3) {
594 if (!line.startsWith("START=")) {
595 throw new ParseException("Failed to parse FFMETADATA: CHAPTER START field missing", lineNumber);
596 }
597
598 String[] startSplit = line.split("=");
599
600 if (startSplit.length != 2) {
601 throw new ParseException("Failed to parse FFMETADATA:"
602 + " CHAPTER START line not correctly formatted", lineNumber);
603 }
604
605 try {
606 start = Long.parseLong(startSplit[1]);
607 }
608 catch (NumberFormatException e) {
609 throw new ParseException("Failed to parse FFMETADATA:"
610 + " CHAPTER START needs to be an Integer", lineNumber);
611 }
612
613 state++;
614 }
615 else if (state == 4) {
616 if (!line.startsWith("END=")) {
617 throw new ParseException("Failed to parse FFMETADATA: CHAPTER END field missing", lineNumber);
618 }
619
620 String[] endSplit = line.split("=");
621
622 if (endSplit.length != 2) {
623 throw new ParseException("Failed to parse FFMETADATA:"
624 + " CHAPTER END line not correctly formatted", lineNumber);
625 }
626
627 try {
628 end = Long.parseLong(endSplit[1]);
629 }
630 catch (NumberFormatException e) {
631 throw new ParseException("Failed to parse FFMETADATA:"
632 + " CHAPTER START needs to be an Integer", lineNumber);
633 }
634
635 state++;
636 }
637
638 else if (state == 5) {
639 if (!line.startsWith("title=")) {
640 state = 7;
641 continue;
642 }
643
644 String fakeLine = Arrays.stream(line.split("="))
645 .skip(1)
646 .collect(Collectors.joining());
647
648 title = Optional.of(new StringBuilder());
649
650
651 line = fakeLine;
652 state++;
653 continue;
654 }
655
656 else if (state == 6) {
657 int[] codePoints = line.codePoints().toArray();
658 boolean isEscaped = false;
659 for (int codePoint : codePoints) {
660 if (isEscaped) {
661 title.get().appendCodePoint(codePoint);
662
663 isEscaped = false;
664 }
665 else {
666 if (codePoint == "\\".codePointAt(0)) {
667 isEscaped = true;
668 }
669 else if (
670 codePoint == "=".codePointAt(0)
671 || codePoint == ";".codePointAt(0)
672 || codePoint == "#".codePointAt(0)
673 ) {
674 throw new ParseException("Failed to parse FFMETADATA:"
675 + " CHAPTER title field '=' ';' '#' '\\' '\\n' have to be escaped", lineNumber);
676 }
677 else {
678 title.get().appendCodePoint(codePoint);
679 }
680 }
681 }
682
683 if (!isEscaped) {
684 state++;
685 }
686 }
687 else if (state == 7) {
688 state = 1;
689
690 Chapter chapter = new Chapter();
691 chapter.title = title.map((t) -> t.toString());
692 chapter.start = timebase * start;
693 chapter.end = timebase * end;
694
695 chapters.add(chapter);
696
697 timebase = defaultTimebase;
698 start = -1;
699 end = -1;
700 title = Optional.empty();
701
702 if (line == null) {
703 break;
704 }
705 continue;
706 }
707
708 line = reader.readLine();
709 lineNumber++;
710
711 if (line == null) {
712 if (state <= 1) {
713
714
715 break;
716 }
717
718 else if (state == 5 || state == 7) {
719 state = 7;
720 }
721 else {
722 throw new ParseException("Failed to parse FFMETADATA: Unexpected end of file", lineNumber);
723 }
724 }
725 }
726
727 return chapters;
728 }
729
730 private Mpeg7Catalog segmentFromChapter(final List<Chapter> chapters, final Track track) {
731 Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
732
733
734 MediaTime contentTime = new MediaRelTimeImpl(0,
735 track.getDuration());
736 MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
737 Video videoContent = mpeg7.addVideoContent("videosegment",
738 contentTime, contentLocator);
739
740 int segmentNum = 0;
741 for (Chapter chapter : chapters) {
742 segmentNum++;
743
744 Segment s = videoContent.getTemporalDecomposition()
745 .createSegment("segment-" + segmentNum);
746
747 s.setMediaTime(new MediaRelTimeImpl((long) (chapter.start * 1000), (long) (chapter.end * 1000)));
748 }
749
750 return mpeg7;
751 }
752
753 private Mpeg7Catalog segmentAndOptimize(final Track track, final File mediaFile, final URL mediaUrl)
754 throws IOException, VideoSegmenterException {
755 Mpeg7Catalog mpeg7 = null;
756
757 MediaTime contentTime = new MediaRelTimeImpl(0,
758 track.getDuration());
759 MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
760
761 Video videoContent;
762
763 logger.debug("changesThreshold: {}, stabilityThreshold: {}", changesThreshold, stabilityThreshold);
764 logger.debug("prefNumber: {}, maxCycles: {}", prefNumber, maxCycles);
765
766 boolean endOptimization = false;
767 int cycleCount = 0;
768 LinkedList<Segment> segments;
769 LinkedList<OptimizationStep> optimizationList = new LinkedList<OptimizationStep>();
770 LinkedList<OptimizationStep> unusedResultsList = new LinkedList<OptimizationStep>();
771 OptimizationStep stepBest = new OptimizationStep();
772
773
774 float changesThresholdLocal = changesThreshold;
775
776
777 int prefNumberLocal = prefNumber;
778 int absoluteMaxLocal = absoluteMax;
779 int absoluteMinLocal = absoluteMin;
780
781
782
783 if (durationDependent) {
784 double trackDurationInHours = track.getDuration() / 3600000.0;
785 prefNumberLocal = (int) Math.round(trackDurationInHours * prefNumberLocal);
786 absoluteMaxLocal = (int) Math.round(trackDurationInHours * absoluteMax);
787 absoluteMinLocal = (int) Math.round(trackDurationInHours * absoluteMin);
788
789
790 if (prefNumberLocal <= 0) {
791 prefNumberLocal = 1;
792 }
793
794 logger.info("Numbers of segments are set to be relative to track duration. Therefore for {} the preferred "
795 + "number of segments is {}", mediaUrl, prefNumberLocal);
796 }
797
798 logger.info("Starting video segmentation of {}", mediaUrl);
799
800
801
802
803 while (!endOptimization) {
804
805 mpeg7 = mpeg7CatalogService.newInstance();
806 videoContent = mpeg7.addVideoContent("videosegment",
807 contentTime, contentLocator);
808
809
810
811 segments = runSegmentationFFmpeg(track, videoContent, mediaFile, changesThresholdLocal);
812
813
814
815
816
817 OptimizationStep currentStep = new OptimizationStep(changesThresholdLocal, segments.size(), prefNumberLocal,
818 mpeg7, segments);
819
820 LinkedList<Segment> segmentsNew = new LinkedList<Segment>();
821 OptimizationStep currentStepFiltered = new OptimizationStep(
822 changesThresholdLocal, 0,
823 prefNumberLocal, filterSegmentation(segments, track, segmentsNew, stabilityThreshold * 1000), segments);
824 currentStepFiltered.setSegmentNumAndRecalcErrors(segmentsNew.size());
825
826 logger.info("Segmentation yields {} segments after filtering", segmentsNew.size());
827
828 OptimizationStep currentStepBest;
829
830
831
832
833
834
835
836
837
838
839
840 if (currentStep.getErrorAbs() <= currentStepFiltered.getErrorAbs() || (segmentsNew.size() < prefNumberLocal
841 && currentStep.getSegmentNum() > (track.getDuration() / 1000.0f) / (stabilityThreshold / 2)
842 && !(currentStepFiltered.getErrorAbs() <= maxError))) {
843
844 optimizationList.add(currentStep);
845 Collections.sort(optimizationList);
846 currentStepBest = currentStep;
847 unusedResultsList.add(currentStepFiltered);
848 } else {
849 optimizationList.add(currentStepFiltered);
850 Collections.sort(optimizationList);
851 currentStepBest = currentStepFiltered;
852 }
853
854 cycleCount++;
855
856 logger.debug("errorAbs = {}, error = {}", currentStep.getErrorAbs(), currentStep.getError());
857 logger.debug("changesThreshold = {}", changesThresholdLocal);
858 logger.debug("cycleCount = {}", cycleCount);
859
860
861 if (cycleCount >= maxCycles || currentStepBest.getErrorAbs() <= maxError) {
862 endOptimization = true;
863 if (optimizationList.size() > 0) {
864 if (optimizationList.getFirst().getErrorAbs() <= optimizationList.getLast().getErrorAbs()
865 && optimizationList.getFirst().getError() >= 0) {
866 stepBest = optimizationList.getFirst();
867 } else {
868 stepBest = optimizationList.getLast();
869 }
870 }
871
872
873 for (OptimizationStep currentUnusedStep : unusedResultsList) {
874 if (currentUnusedStep.getErrorAbs() < stepBest.getErrorAbs()) {
875 stepBest = unusedResultsList.getFirst();
876 }
877 }
878
879
880
881 } else {
882 OptimizationStep first = optimizationList.getFirst();
883 OptimizationStep last = optimizationList.getLast();
884
885
886 if (optimizationList.size() == 1 || first.getError() < 0 || last.getError() > 0) {
887 if (currentStepBest.getError() >= 0) {
888
889 if (currentStepBest.getError() <= 1) {
890 changesThresholdLocal += changesThresholdLocal * currentStepBest.getError();
891 } else {
892
893
894 if (cycleCount <= 1 && currentStep.getSegmentNum() > 2000) {
895 changesThresholdLocal = 0.2f;
896
897
898 } else {
899 changesThresholdLocal *= 2;
900 }
901 }
902 } else {
903 changesThresholdLocal /= 2;
904 }
905
906 logger.debug("onesided optimization yields new changesThreshold = {}", changesThresholdLocal);
907
908 } else {
909
910
911
912
913
914
915
916 float x = (first.getSegmentNum() - prefNumberLocal) / (float) (first.getSegmentNum() - last.getSegmentNum());
917 float newX = ((x + 0.5f) * 0.5f);
918 changesThresholdLocal = first.getChangesThreshold() * (1 - newX) + last.getChangesThreshold() * newX;
919 logger.debug("doublesided optimization yields new changesThreshold = {}", changesThresholdLocal);
920 }
921 }
922 }
923
924
925
926
927 int threshLow = stabilityThreshold * 1000;
928 int threshHigh = threshLow + (threshLow / 2);
929
930 LinkedList<Segment> tmpSegments;
931 float smallestError = Float.MAX_VALUE;
932 int bestI = threshLow;
933 segments = stepBest.getSegments();
934
935
936
937 if (stepBest.getError() <= maxError) {
938 threshHigh = stabilityThreshold * 1000;
939 }
940 for (int i = threshLow; i <= threshHigh; i = i + 1000) {
941 tmpSegments = new LinkedList<Segment>();
942 filterSegmentation(segments, track, tmpSegments, i);
943 float newError = OptimizationStep.calculateErrorAbs(tmpSegments.size(), prefNumberLocal);
944 if (newError < smallestError) {
945 smallestError = newError;
946 bestI = i;
947 }
948 }
949 tmpSegments = new LinkedList<Segment>();
950 mpeg7 = filterSegmentation(segments, track, tmpSegments, bestI);
951
952
953 logger.debug("result segments:");
954 for (int i = 0; i < tmpSegments.size(); i++) {
955 int[] tmpLog2 = new int[7];
956 tmpLog2[0] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getHour();
957 tmpLog2[1] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getMinutes();
958 tmpLog2[2] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getSeconds();
959 tmpLog2[3] = tmpSegments.get(i).getMediaTime().getMediaDuration().getHours();
960 tmpLog2[4] = tmpSegments.get(i).getMediaTime().getMediaDuration().getMinutes();
961 tmpLog2[5] = tmpSegments.get(i).getMediaTime().getMediaDuration().getSeconds();
962 Object[] tmpLog1 = {tmpLog2[0], tmpLog2[1], tmpLog2[2], tmpLog2[3], tmpLog2[4], tmpLog2[5], tmpLog2[6]};
963 tmpLog1[6] = tmpSegments.get(i).getIdentifier();
964 logger.debug("s:{}:{}:{}, d:{}:{}:{}, {}", tmpLog1);
965 }
966
967 logger.info("Optimized Segmentation yields (after {} iteration" + (cycleCount == 1 ? "" : "s") + ") {} segments",
968 cycleCount, tmpSegments.size());
969
970
971 if (tmpSegments.size() < absoluteMinLocal || tmpSegments.size() > absoluteMaxLocal) {
972 mpeg7 = uniformSegmentation(track, tmpSegments, prefNumberLocal);
973 logger.info("Since no reasonable segmentation could be found, a uniform segmentation was created");
974 }
975
976 return mpeg7;
977 }
978
979
980
981
982
983
984
985
986
987
988
989
990
991 private LinkedList<Segment> runSegmentationFFmpeg(Track track, Video videoContent, File mediaFile,
992 float changesThreshold) throws IOException, VideoSegmenterException {
993
994 String[] command = new String[] {
995 binary,
996 "-nostats", "-nostdin",
997 "-i", mediaFile.getAbsolutePath(),
998 "-filter:v", "select=gt(scene\\," + changesThreshold + "),showinfo",
999 "-f", "null",
1000 "-"
1001 };
1002
1003 logger.info("Detecting video segments using command: {}", (Object) command);
1004
1005 ProcessBuilder pbuilder = new ProcessBuilder(command);
1006 List<String> segmentsStrings = new LinkedList<>();
1007 Process process = pbuilder.start();
1008 try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
1009 String line = reader.readLine();
1010 while (null != line) {
1011 if (line.startsWith("[Parsed_showinfo")) {
1012 segmentsStrings.add(line);
1013 }
1014 line = reader.readLine();
1015 }
1016 } catch (IOException e) {
1017 logger.error("Error executing ffmpeg: {}", e.getMessage());
1018 }
1019
1020
1021
1022
1023
1024 int segmentcount = 1;
1025 LinkedList<Segment> segments = new LinkedList<>();
1026
1027 if (segmentsStrings.size() == 0) {
1028 Segment s = videoContent.getTemporalDecomposition()
1029 .createSegment("segment-" + segmentcount);
1030 s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
1031 segments.add(s);
1032 } else {
1033 long starttime = 0;
1034 long endtime = 0;
1035 Pattern pattern = Pattern.compile("pts_time\\:\\d+(\\.\\d+)?");
1036 for (String seginfo : segmentsStrings) {
1037 Matcher matcher = pattern.matcher(seginfo);
1038 String time = "";
1039 while (matcher.find()) {
1040 time = matcher.group().substring(9);
1041 }
1042 if ("".equals(time)) {
1043
1044
1045 continue;
1046 }
1047 try {
1048 endtime = Math.round(Float.parseFloat(time) * 1000);
1049 } catch (NumberFormatException e) {
1050 logger.error("Unable to parse FFmpeg output, likely FFmpeg version mismatch!", e);
1051 throw new VideoSegmenterException(e);
1052 }
1053 long segmentLength = endtime - starttime;
1054 if (1000 * stabilityThresholdPrefilter < segmentLength) {
1055 Segment segment = videoContent.getTemporalDecomposition()
1056 .createSegment("segment-" + segmentcount);
1057 segment.setMediaTime(new MediaRelTimeImpl(starttime,
1058 endtime - starttime));
1059 logger.debug("Created segment {} at start time {} with duration {}", segmentcount, starttime, endtime);
1060 segments.add(segment);
1061 segmentcount++;
1062 starttime = endtime;
1063 }
1064 }
1065
1066 Segment s = videoContent.getTemporalDecomposition()
1067 .createSegment("segment-" + segmentcount);
1068 s.setMediaTime(new MediaRelTimeImpl(starttime, track.getDuration() - starttime));
1069 logger.debug("Created segment {} at start time {} with duration {}", segmentcount, starttime,
1070 track.getDuration() - endtime);
1071 segments.add(s);
1072 }
1073
1074 logger.info("Segmentation of {} yields {} segments",
1075 mediaFile.toURI().toURL(), segments.size());
1076
1077 return segments;
1078 }
1079
1080
1081
1082
1083
1084
1085 @Override
1086 protected String process(Job job) throws Exception {
1087 Operation op = null;
1088 String operation = job.getOperation();
1089 List<String> arguments = job.getArguments();
1090 try {
1091 op = Operation.valueOf(operation);
1092 switch (op) {
1093 case Segment:
1094 Track track = (Track) MediaPackageElementParser
1095 .getFromXml(arguments.get(0));
1096 Catalog catalog = segment(job, track);
1097 return MediaPackageElementParser.getAsXml(catalog);
1098 default:
1099 throw new IllegalStateException(
1100 "Don't know how to handle operation '" + operation
1101 + "'");
1102 }
1103 } catch (IllegalArgumentException e) {
1104 throw new ServiceRegistryException(
1105 "This service can't handle operations of type '" + op + "'",
1106 e);
1107 } catch (IndexOutOfBoundsException e) {
1108 throw new ServiceRegistryException(
1109 "This argument list for operation '" + op
1110 + "' does not meet expectations", e);
1111 } catch (Exception e) {
1112 throw new ServiceRegistryException("Error handling operation '"
1113 + op + "'", e);
1114 }
1115 }
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125 protected Mpeg7Catalog filterSegmentation(
1126 LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew) {
1127 int mergeThresh = stabilityThreshold * 1000;
1128 return filterSegmentation(segments, track, segmentsNew, mergeThresh);
1129 }
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141 protected Mpeg7Catalog filterSegmentation(
1142 LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew, int mergeThresh) {
1143 if (segmentsNew == null) {
1144 segmentsNew = new LinkedList<Segment>();
1145 }
1146 boolean merging = false;
1147 MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
1148 MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
1149 Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
1150 Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
1151
1152 int segmentcount = 1;
1153
1154 MediaTimePoint currentSegStart = new MediaTimePointImpl();
1155
1156 for (Segment o : segments) {
1157
1158
1159 if (o.getMediaTime().getMediaDuration().getDurationInMilliseconds() <= mergeThresh) {
1160
1161 if (!merging) {
1162 currentSegStart = o.getMediaTime().getMediaTimePoint();
1163 merging = true;
1164 }
1165
1166
1167 } else {
1168 long currentSegDuration = o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
1169 long currentSegEnd = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds()
1170 + currentSegDuration;
1171
1172 if (merging) {
1173 long newDuration = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds()
1174 - currentSegStart.getTimeInMilliseconds();
1175
1176
1177
1178 if (newDuration >= mergeThresh) {
1179 Segment s = videoContent.getTemporalDecomposition()
1180 .createSegment("segment-" + segmentcount++);
1181 s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
1182 segmentsNew.add(s);
1183
1184
1185 Segment s2 = videoContent.getTemporalDecomposition()
1186 .createSegment("segment-" + segmentcount++);
1187 s2.setMediaTime(o.getMediaTime());
1188 segmentsNew.add(s2);
1189
1190
1191
1192 } else {
1193 long followingStartOld = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
1194 long newSplit = (currentSegStart.getTimeInMilliseconds() + followingStartOld) / 2;
1195 long followingEnd = followingStartOld + o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
1196 long followingDuration = followingEnd - newSplit;
1197
1198
1199 if (segmentsNew.isEmpty()) {
1200 Segment s = videoContent.getTemporalDecomposition()
1201 .createSegment("segment-" + segmentcount++);
1202 s.setMediaTime(new MediaRelTimeImpl(0, followingEnd));
1203 segmentsNew.add(s);
1204 } else {
1205
1206 long previousStart = segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
1207
1208
1209 segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(previousStart, newSplit - previousStart));
1210
1211
1212 Segment s = videoContent.getTemporalDecomposition()
1213 .createSegment("segment-" + segmentcount++);
1214 s.setMediaTime(new MediaRelTimeImpl(newSplit, followingDuration));
1215 segmentsNew.add(s);
1216 }
1217 }
1218 merging = false;
1219
1220
1221 } else {
1222 Segment s = videoContent.getTemporalDecomposition()
1223 .createSegment("segment-" + segmentcount++);
1224 s.setMediaTime(o.getMediaTime());
1225 segmentsNew.add(s);
1226 }
1227 }
1228 }
1229
1230
1231 if (merging && !segmentsNew.isEmpty()) {
1232
1233 long newDuration = track.getDuration() - currentSegStart.getTimeInMilliseconds();
1234
1235 if (newDuration >= mergeThresh) {
1236
1237 Segment s = videoContent.getTemporalDecomposition()
1238 .createSegment("segment-" + segmentcount);
1239 s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
1240 segmentsNew.add(s);
1241
1242
1243 } else {
1244 newDuration = track.getDuration() - segmentsNew.getLast().getMediaTime().getMediaTimePoint()
1245 .getTimeInMilliseconds();
1246 segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(segmentsNew.getLast().getMediaTime()
1247 .getMediaTimePoint().getTimeInMilliseconds(), newDuration));
1248
1249 }
1250 }
1251
1252
1253
1254 if (segmentsNew.isEmpty()) {
1255 Segment s = videoContent.getTemporalDecomposition()
1256 .createSegment("segment-" + segmentcount);
1257 s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
1258 segmentsNew.add(s);
1259 }
1260
1261 return mpeg7;
1262 }
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273 protected Mpeg7Catalog uniformSegmentation(Track track, LinkedList<Segment> segmentsNew, int prefNumber) {
1274 if (segmentsNew == null) {
1275 segmentsNew = new LinkedList<Segment>();
1276 }
1277 MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
1278 MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
1279 Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
1280 Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
1281
1282 long segmentDuration = track.getDuration() / prefNumber;
1283 long currentSegStart = 0;
1284
1285
1286 for (int i = 1; i < prefNumber; i++) {
1287 Segment s = videoContent.getTemporalDecomposition()
1288 .createSegment("segment-" + i);
1289 s.setMediaTime(new MediaRelTimeImpl(currentSegStart, segmentDuration));
1290 segmentsNew.add(s);
1291
1292 currentSegStart += segmentDuration;
1293 }
1294
1295
1296 Segment s = videoContent.getTemporalDecomposition()
1297 .createSegment("segment-" + prefNumber);
1298 s.setMediaTime(new MediaRelTimeImpl(currentSegStart, track.getDuration() - currentSegStart));
1299 segmentsNew.add(s);
1300
1301 return mpeg7;
1302 }
1303
1304
1305
1306
1307
1308
1309
1310 @Reference
1311 protected void setWorkspace(Workspace workspace) {
1312 this.workspace = workspace;
1313 }
1314
1315
1316
1317
1318
1319
1320
1321 @Reference(name = "Mpeg7Service")
1322 protected void setMpeg7CatalogService(
1323 Mpeg7CatalogService mpeg7CatalogService) {
1324 this.mpeg7CatalogService = mpeg7CatalogService;
1325 }
1326
1327
1328
1329
1330
1331
1332
1333 @Reference
1334 protected void setServiceRegistry(ServiceRegistry serviceRegistry) {
1335 this.serviceRegistry = serviceRegistry;
1336 }
1337
1338
1339
1340
1341
1342
1343 @Override
1344 protected ServiceRegistry getServiceRegistry() {
1345 return serviceRegistry;
1346 }
1347
1348
1349
1350
1351
1352
1353
1354 @Reference
1355 public void setSecurityService(SecurityService securityService) {
1356 this.securityService = securityService;
1357 }
1358
1359
1360
1361
1362
1363
1364
1365 @Reference
1366 public void setUserDirectoryService(
1367 UserDirectoryService userDirectoryService) {
1368 this.userDirectoryService = userDirectoryService;
1369 }
1370
1371
1372
1373
1374
1375
1376
1377 @Reference
1378 public void setOrganizationDirectoryService(
1379 OrganizationDirectoryService organizationDirectory) {
1380 this.organizationDirectoryService = organizationDirectory;
1381 }
1382
1383
1384
1385
1386
1387
1388 @Override
1389 protected SecurityService getSecurityService() {
1390 return securityService;
1391 }
1392
1393
1394
1395
1396
1397
1398 @Override
1399 protected UserDirectoryService getUserDirectoryService() {
1400 return userDirectoryService;
1401 }
1402
1403
1404
1405
1406
1407
1408 @Override
1409 protected OrganizationDirectoryService getOrganizationDirectoryService() {
1410 return organizationDirectoryService;
1411 }
1412
1413 }