VideoSegmenterServiceImpl.java
/*
* Licensed to The Apereo Foundation under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
*
* The Apereo Foundation licenses this file to you under the Educational
* Community License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License
* at:
*
* http://opensource.org/licenses/ecl2.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package org.opencastproject.videosegmenter.ffmpeg;
import org.opencastproject.job.api.AbstractJobProducer;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.Catalog;
import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageElements;
import org.opencastproject.mediapackage.MediaPackageException;
import org.opencastproject.mediapackage.Track;
import org.opencastproject.metadata.mpeg7.MediaLocator;
import org.opencastproject.metadata.mpeg7.MediaLocatorImpl;
import org.opencastproject.metadata.mpeg7.MediaRelTimeImpl;
import org.opencastproject.metadata.mpeg7.MediaTime;
import org.opencastproject.metadata.mpeg7.MediaTimePoint;
import org.opencastproject.metadata.mpeg7.MediaTimePointImpl;
import org.opencastproject.metadata.mpeg7.Mpeg7Catalog;
import org.opencastproject.metadata.mpeg7.Mpeg7CatalogService;
import org.opencastproject.metadata.mpeg7.Segment;
import org.opencastproject.metadata.mpeg7.Video;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.LoadUtil;
import org.opencastproject.util.MimeType;
import org.opencastproject.util.MimeTypes;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.videosegmenter.api.VideoSegmenterException;
import org.opencastproject.videosegmenter.api.VideoSegmenterService;
import org.opencastproject.workspace.api.Workspace;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.osgi.service.cm.ConfigurationException;
import org.osgi.service.cm.ManagedService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URL;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Dictionary;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Media analysis plugin that takes a video stream and extracts video segments
* by trying to detect slide and/or scene changes.
*
* This plugin runs
*
* <pre>
* ffmpeg -nostats -i in.mp4 -filter:v 'select=gt(scene\,0.04),showinfo' -f null - 2>&1 | grep Parsed_showinfo_1
* </pre>
*/
@Component(
immediate = true,
service = { VideoSegmenterService.class,ManagedService.class },
property = {
"service.description=VideoSegmenter Service"
}
)
public class VideoSegmenterServiceImpl extends AbstractJobProducer implements
VideoSegmenterService, ManagedService {
/** Resulting collection in the working file repository */
public static final String COLLECTION_ID = "videosegments";
/** List of available operations on jobs */
private enum Operation {
Segment
};
private class Chapter {
protected double start;
protected double end;
protected Optional<String> title;
};
/** Path to the executable */
protected String binary;
public static final String FFMPEG_BINARY_CONFIG = "org.opencastproject.composer.ffmpeg.path";
public static final String FFMPEG_BINARY_DEFAULT = "ffmpeg";
/** Name of the constant used to retrieve the stability threshold */
public static final String OPT_STABILITY_THRESHOLD = "stabilitythreshold";
/** The number of seconds that need to resemble until a scene is considered "stable" */
public static final int DEFAULT_STABILITY_THRESHOLD = 60;
/** Name of the constant used to retrieve the changes threshold */
public static final String OPT_CHANGES_THRESHOLD = "changesthreshold";
/** Default value for the number of pixels that may change between two frames without considering them different */
public static final float DEFAULT_CHANGES_THRESHOLD = 0.025f; // 2.5% change
/** Name of the constant used to retrieve the preferred number of segments */
public static final String OPT_PREF_NUMBER = "prefNumber";
/** Default value for the preferred number of segments */
public static final int DEFAULT_PREF_NUMBER = 30;
/** Name of the constant used to retrieve the maximum number of cycles */
public static final String OPT_MAX_CYCLES = "maxCycles";
/** Default value for the maximum number of cycles */
public static final int DEFAULT_MAX_CYCLES = 3;
/** Name of the constant used to retrieve the maximum tolerance for result */
public static final String OPT_MAX_ERROR = "maxError";
/** Default value for the maximum tolerance for result */
public static final float DEFAULT_MAX_ERROR = 0.25f;
/** Name of the constant used to retrieve the absolute maximum number of segments */
public static final String OPT_ABSOLUTE_MAX = "absoluteMax";
/** Default value for the absolute maximum number of segments */
public static final int DEFAULT_ABSOLUTE_MAX = 150;
/** Name of the constant used to retrieve the absolute minimum number of segments */
public static final String OPT_ABSOLUTE_MIN = "absoluteMin";
/** Default value for the absolute minimum number of segments */
public static final int DEFAULT_ABSOLUTE_MIN = 3;
/** Name of the constant used to retrieve the option whether segments numbers depend on track duration */
public static final String OPT_DURATION_DEPENDENT = "durationDependent";
/** Default value for the option whether segments numbers depend on track duration */
public static final boolean DEFAULT_DURATION_DEPENDENT = false;
/** Name of the configuration option deciding whether the chapter extraction is used for segmentation */
public static final String OPT_USE_CHAPTER_IF_AVAILABLE = "useChapterIfAvailable";
/** Default value for the chapter extraction option */
public static final boolean DEFAULT_USE_CHAPTER_IF_AVAILABLE = false;
private boolean useChapterIfAvailable = DEFAULT_USE_CHAPTER_IF_AVAILABLE;
/** Name of the configuration option deciding which tracks should have their chapters extracted based on mime type */
public static final String OPT_USE_CHAPTER_MIME_TYPES = "useChapterMimeTypes";
public static final List<MimeType> DEFAULT_USE_CHAPTER_MIME_TYPES = new ArrayList<>();
private List<MimeType> useChapterMimeTypes = DEFAULT_USE_CHAPTER_MIME_TYPES;
/** The load introduced on the system by a segmentation job */
public static final float DEFAULT_SEGMENTER_JOB_LOAD = 0.3f;
/** The key to look for in the service configuration file to override the DEFAULT_CAPTION_JOB_LOAD */
public static final String SEGMENTER_JOB_LOAD_KEY = "job.load.videosegmenter";
/** The load introduced on the system by creating a caption job */
private float segmenterJobLoad = DEFAULT_SEGMENTER_JOB_LOAD;
/** The logging facility */
protected static final Logger logger = LoggerFactory
.getLogger(VideoSegmenterServiceImpl.class);
/** Number of pixels that may change between two frames without considering them different */
protected float changesThreshold = DEFAULT_CHANGES_THRESHOLD;
/** The number of seconds that need to resemble until a scene is considered "stable" */
protected int stabilityThreshold = DEFAULT_STABILITY_THRESHOLD;
/** The minimum segment length in seconds for creation of segments from ffmpeg output */
protected int stabilityThresholdPrefilter = 1;
/** The number of segments that should be generated */
protected int prefNumber = DEFAULT_PREF_NUMBER;
/** The number of cycles after which the optimization of the number of segments is forced to end */
protected int maxCycles = DEFAULT_MAX_CYCLES;
/** The tolerance with which the optimization of the number of segments is considered successful */
protected float maxError = DEFAULT_MAX_ERROR;
/** The absolute maximum for the number of segments whose compliance will be enforced after the optimization*/
protected int absoluteMax = DEFAULT_ABSOLUTE_MAX;
/** The absolute minimum for the number of segments whose compliance will be enforced after the optimization*/
protected int absoluteMin = DEFAULT_ABSOLUTE_MIN;
/** The boolean that defines whether segment numbers are interpreted as absolute or relative to track duration */
protected boolean durationDependent = DEFAULT_DURATION_DEPENDENT;
/** Reference to the receipt service */
protected ServiceRegistry serviceRegistry = null;
/** The mpeg-7 service */
protected Mpeg7CatalogService mpeg7CatalogService = null;
/** The workspace to use when retrieving remote media files */
protected Workspace workspace = null;
/** The security service */
protected SecurityService securityService = null;
/** The user directory service */
protected UserDirectoryService userDirectoryService = null;
/** The organization directory service */
protected OrganizationDirectoryService organizationDirectoryService = null;
/**
* Creates a new instance of the video segmenter service.
*/
public VideoSegmenterServiceImpl() {
super(JOB_TYPE);
this.binary = FFMPEG_BINARY_DEFAULT;
}
@Override
public void activate(ComponentContext cc) {
super.activate(cc);
/* Configure segmenter */
final String path = cc.getBundleContext().getProperty(FFMPEG_BINARY_CONFIG);
this.binary = path == null ? FFMPEG_BINARY_DEFAULT : path;
logger.debug("Configuration {}: {}", FFMPEG_BINARY_CONFIG, FFMPEG_BINARY_DEFAULT);
}
/**
* {@inheritDoc}
*
* @see org.osgi.service.cm.ManagedService#updated(java.util.Dictionary)
*/
@Override
public void updated(Dictionary<String, ?> properties) throws ConfigurationException {
if (properties == null) {
return;
}
logger.debug("Configuring the videosegmenter");
// Stability threshold
if (properties.get(OPT_STABILITY_THRESHOLD) != null) {
String threshold = (String) properties.get(OPT_STABILITY_THRESHOLD);
try {
stabilityThreshold = Integer.parseInt(threshold);
logger.info("Stability threshold set to {} consecutive frames", stabilityThreshold);
} catch (Exception e) {
throw new ConfigurationException(OPT_STABILITY_THRESHOLD,
String.format("Found illegal value '%s'", threshold)
);
}
}
// Changes threshold
if (properties.get(OPT_CHANGES_THRESHOLD) != null) {
String threshold = (String) properties.get(OPT_CHANGES_THRESHOLD);
try {
changesThreshold = Float.parseFloat(threshold);
logger.info("Changes threshold set to {}", changesThreshold);
} catch (Exception e) {
throw new ConfigurationException(OPT_CHANGES_THRESHOLD,
String.format("Found illegal value '%s'", threshold)
);
}
}
// Preferred Number of Segments
if (properties.get(OPT_PREF_NUMBER) != null) {
String number = (String) properties.get(OPT_PREF_NUMBER);
try {
prefNumber = Integer.parseInt(number);
logger.info("Preferred number of segments set to {}", prefNumber);
} catch (Exception e) {
throw new ConfigurationException(OPT_PREF_NUMBER,
String.format("Found illegal value '%s'", number)
);
}
}
// Maximum number of cycles
if (properties.get(OPT_MAX_CYCLES) != null) {
String number = (String) properties.get(OPT_MAX_CYCLES);
try {
maxCycles = Integer.parseInt(number);
logger.info("Maximum number of cycles set to {}", maxCycles);
} catch (Exception e) {
throw new ConfigurationException(OPT_MAX_CYCLES,
String.format("Found illegal value '%s'", number)
);
}
}
// Absolute maximum number of segments
if (properties.get(OPT_ABSOLUTE_MAX) != null) {
String number = (String) properties.get(OPT_ABSOLUTE_MAX);
try {
absoluteMax = Integer.parseInt(number);
logger.info("Absolute maximum number of segments set to {}", absoluteMax);
} catch (Exception e) {
throw new ConfigurationException(OPT_ABSOLUTE_MAX,
String.format("Found illegal value '%s'", number)
);
}
}
// Absolute minimum number of segments
if (properties.get(OPT_ABSOLUTE_MIN) != null) {
String number = (String) properties.get(OPT_ABSOLUTE_MIN);
try {
absoluteMin = Integer.parseInt(number);
logger.info("Absolute minimum number of segments set to {}", absoluteMin);
} catch (Exception e) {
throw new ConfigurationException(OPT_ABSOLUTE_MIN,
String.format("Found illegal value '%s'", number)
);
}
}
// Dependency on video duration
if (properties.get(OPT_DURATION_DEPENDENT) != null) {
String value = (String) properties.get(OPT_DURATION_DEPENDENT);
try {
durationDependent = BooleanUtils.toBooleanObject(StringUtils.trimToNull(value));
logger.info("Dependency on video duration is set to {}", durationDependent);
} catch (Exception e) {
throw new ConfigurationException(OPT_DURATION_DEPENDENT,
String.format("Found illegal value '%s'", value)
);
}
}
if (properties.get(OPT_USE_CHAPTER_IF_AVAILABLE) != null) {
String value = (String) properties.get(OPT_USE_CHAPTER_IF_AVAILABLE);
try {
useChapterIfAvailable = BooleanUtils.toBooleanObject(StringUtils.trimToNull(value));
logger.info("Use Chapters if available is set to {}", useChapterIfAvailable);
} catch (Exception e) {
throw new ConfigurationException(OPT_USE_CHAPTER_IF_AVAILABLE,
String.format("Found illegal value '%s'", value)
);
}
}
if (properties.get(OPT_USE_CHAPTER_MIME_TYPES) != null) {
String value = (String) properties.get(OPT_USE_CHAPTER_MIME_TYPES);
try {
List<MimeType> mts = new ArrayList<>();
String[] values = value.split(",");
for (String mimeString : values) {
MimeType mt = MimeTypes.parseMimeType(mimeString);
mts.add(mt);
}
useChapterMimeTypes = mts;
} catch (Exception e) {
throw new ConfigurationException(OPT_USE_CHAPTER_MIME_TYPES,
String.format("Found illegal value '%s'", value)
);
}
} else {
useChapterMimeTypes = DEFAULT_USE_CHAPTER_MIME_TYPES;
}
segmenterJobLoad = LoadUtil.getConfiguredLoadValue(
properties, SEGMENTER_JOB_LOAD_KEY, DEFAULT_SEGMENTER_JOB_LOAD, serviceRegistry);
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.videosegmenter.api.VideoSegmenterService#segment(org.opencastproject.mediapackage.Track)
*/
public Job segment(Track track) throws VideoSegmenterException,
MediaPackageException {
try {
return serviceRegistry.createJob(JOB_TYPE,
Operation.Segment.toString(),
Arrays.asList(MediaPackageElementParser.getAsXml(track)), segmenterJobLoad);
} catch (ServiceRegistryException e) {
throw new VideoSegmenterException("Unable to create a job", e);
}
}
/**
* Starts segmentation on the video track identified by
* <code>mediapackageId</code> and <code>elementId</code> and returns a
* receipt containing the final result in the form of anMpeg7Catalog.
*
* @param track
* the element to analyze
* @return a receipt containing the resulting mpeg-7 catalog
* @throws VideoSegmenterException
*/
protected Catalog segment(Job job, Track track)
throws VideoSegmenterException, MediaPackageException {
// Make sure the element can be analyzed using this analysis
// implementation
if (!track.hasVideo()) {
logger.warn("Element {} is not a video track", track);
throw new VideoSegmenterException("Element is not a video track");
}
try {
File mediaFile = null;
URL mediaUrl = null;
try {
mediaFile = workspace.get(track.getURI());
mediaUrl = mediaFile.toURI().toURL();
} catch (NotFoundException e) {
throw new VideoSegmenterException(
"Error finding the video file in the workspace", e);
} catch (IOException e) {
throw new VideoSegmenterException(
"Error reading the video file in the workspace", e);
}
if (track.getDuration() == null) {
throw new MediaPackageException("Track " + track
+ " does not have a duration");
}
logger.info("Track {} loaded, duration is {} s", mediaUrl,
track.getDuration() / 1000);
Mpeg7Catalog mpeg7;
Optional<List<Chapter>> chapter = Optional.empty();
if (useChapterIfAvailable
&& (useChapterMimeTypes.isEmpty()
|| useChapterMimeTypes.stream().anyMatch(comp -> track.getMimeType().eq(comp)))) {
chapter = Optional.ofNullable(extractChapter(mediaFile));
}
if (chapter.isPresent() && !chapter.get().isEmpty()) {
mpeg7 = segmentFromChapter(chapter.get(), track);
} else {
mpeg7 = segmentAndOptimize(track, mediaFile, mediaUrl);
}
Catalog mpeg7Catalog = (Catalog) MediaPackageElementBuilderFactory
.newInstance().newElementBuilder()
.newElement(Catalog.TYPE, MediaPackageElements.SEGMENTS);
URI uri;
try {
uri = workspace.putInCollection(COLLECTION_ID, job.getId()
+ ".xml", mpeg7CatalogService.serialize(mpeg7));
} catch (IOException e) {
throw new VideoSegmenterException(
"Unable to put the mpeg7 catalog into the workspace", e);
}
mpeg7Catalog.setURI(uri);
logger.info("Finished video segmentation of {}", mediaUrl);
return mpeg7Catalog;
} catch (Exception e) {
logger.warn("Error segmenting " + track, e);
if (e instanceof VideoSegmenterException) {
throw (VideoSegmenterException) e;
} else {
throw new VideoSegmenterException(e);
}
}
}
/**
* Extracts the Chapter information from an container, with the help of ffmpeg
* @param mediaFile the file, which contains the chapter information
* @return The extracted chapters
*/
private List<Chapter> extractChapter(final File mediaFile) throws IOException {
String[] command = new String[] {
binary,
"-nostats", "-nostdin",
"-i", mediaFile.getAbsolutePath(),
"-f", "FFMETADATA",
"-"
};
logger.debug("Detecting chapters using command: {}", (Object) command);
ProcessBuilder pbuilder = new ProcessBuilder(command);
Process process = pbuilder.start();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
return parseChapter(reader);
} catch (IOException e) {
logger.error("Error executing ffmpeg: {}", e.getMessage());
} catch (ParseException e) {
logger.error("Error parsing ffmpeg output: {}", e.getMessage());
}
return null;
}
/**
* Parses Chapter information out of a FFMETADATA file (see https://ffmpeg.org/ffmpeg-formats.html section 5 )
* @param reader The Reader to parse from
* @return List of parsed chapters
* @throws IOException When the reading the reader fails
* @throws ParseException When the parsing of the FFMETADATA reader fails
*/
private List<Chapter> parseChapter(final BufferedReader reader) throws IOException, ParseException {
List<Chapter> chapters = new ArrayList<Chapter>();
int state = 0;
// Nanoseconds are the default timebase
final double defaultTimebase = 1e-9f;
double timebase = defaultTimebase;
long start = -1;
long end = -1;
Optional<StringBuilder> title = Optional.empty();
String line = reader.readLine();
int lineNumber = 1;
if (line == null) {
return chapters;
}
while (true) {
// begin parsing
if (state == 0 && ";FFMETADATA1".equals(line)) {
state++;
}
// ignore comments, empty lines
else if (line != null && (line.startsWith(";") || line.startsWith("#") || line.isEmpty())) { }
// search for chapter begin
else if (state == 1 && "[CHAPTER]".equals(line)) {
state++;
}
// check for timebase
else if (state == 2) {
// timebase is optional
if (!line.startsWith("TIMEBASE=")) {
state++;
continue;
}
String[] timebaseSplit = line.split("=");
if (timebaseSplit.length != 2) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER TIMEBASE line not correctly formatted", lineNumber);
}
String ratio = timebaseSplit[1];
String[] numbers = ratio.split("/");
if (numbers.length != 2) {
throw new ParseException("Failed to parse FFMETADATA: ratio not correctly formatted", lineNumber);
}
try {
// The standard requires Integer here, but this doesn't really matter here
timebase = Double.parseDouble(numbers[0]) / Double.parseDouble(numbers[1]);
}
catch (NumberFormatException e) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " Couldn't parse timebase as ratio of integer numbers", lineNumber);
}
state++;
}
// start point of chapter
else if (state == 3) {
if (!line.startsWith("START=")) {
throw new ParseException("Failed to parse FFMETADATA: CHAPTER START field missing", lineNumber);
}
String[] startSplit = line.split("=");
if (startSplit.length != 2) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER START line not correctly formatted", lineNumber);
}
try {
start = Long.parseLong(startSplit[1]);
}
catch (NumberFormatException e) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER START needs to be an Integer", lineNumber);
}
state++;
}
else if (state == 4) {
if (!line.startsWith("END=")) {
throw new ParseException("Failed to parse FFMETADATA: CHAPTER END field missing", lineNumber);
}
String[] endSplit = line.split("=");
if (endSplit.length != 2) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER END line not correctly formatted", lineNumber);
}
try {
end = Long.parseLong(endSplit[1]);
}
catch (NumberFormatException e) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER START needs to be an Integer", lineNumber);
}
state++;
}
// Being processing of title
else if (state == 5) {
if (!line.startsWith("title=")) {
state = 7;
continue;
}
String fakeLine = Arrays.stream(line.split("="))
.skip(1)
.collect(Collectors.joining());
title = Optional.of(new StringBuilder());
// Process title further in next state
line = fakeLine;
state++;
continue;
}
// Continue processing of title
else if (state == 6) {
int[] codePoints = line.codePoints().toArray();
boolean isEscaped = false;
for (int codePoint : codePoints) {
if (isEscaped) {
title.get().appendCodePoint(codePoint);
isEscaped = false;
}
else {
if (codePoint == "\\".codePointAt(0)) {
isEscaped = true;
}
else if (
codePoint == "=".codePointAt(0)
|| codePoint == ";".codePointAt(0)
|| codePoint == "#".codePointAt(0)
) {
throw new ParseException("Failed to parse FFMETADATA:"
+ " CHAPTER title field '=' ';' '#' '\\' '\\n' have to be escaped", lineNumber);
}
else {
title.get().appendCodePoint(codePoint);
}
}
}
if (!isEscaped) {
state++;
}
}
else if (state == 7) {
state = 1;
Chapter chapter = new Chapter();
chapter.title = title.map((t) -> t.toString());
chapter.start = timebase * start;
chapter.end = timebase * end;
chapters.add(chapter);
timebase = defaultTimebase;
start = -1;
end = -1;
title = Optional.empty();
if (line == null) {
break;
}
continue;
}
line = reader.readLine();
lineNumber++;
if (line == null) {
if (state <= 1) {
// Haven't found a chapter yet or searching for next chapter,
// just finish and return current chapter list
break;
}
// state 5 and 7 can finish up a chapter, continue processing state 7 a last time
else if (state == 5 || state == 7) {
state = 7;
}
else {
throw new ParseException("Failed to parse FFMETADATA: Unexpected end of file", lineNumber);
}
}
}
return chapters;
}
private Mpeg7Catalog segmentFromChapter(final List<Chapter> chapters, final Track track) {
Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
// create videoContent
MediaTime contentTime = new MediaRelTimeImpl(0,
track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Video videoContent = mpeg7.addVideoContent("videosegment",
contentTime, contentLocator);
int segmentNum = 0;
for (Chapter chapter : chapters) {
segmentNum++;
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentNum);
s.setMediaTime(new MediaRelTimeImpl((long) (chapter.start * 1000), (long) (chapter.end * 1000)));
}
return mpeg7;
}
private Mpeg7Catalog segmentAndOptimize(final Track track, final File mediaFile, final URL mediaUrl)
throws IOException, VideoSegmenterException {
Mpeg7Catalog mpeg7 = null;
MediaTime contentTime = new MediaRelTimeImpl(0,
track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Video videoContent;
logger.debug("changesThreshold: {}, stabilityThreshold: {}", changesThreshold, stabilityThreshold);
logger.debug("prefNumber: {}, maxCycles: {}", prefNumber, maxCycles);
boolean endOptimization = false;
int cycleCount = 0;
LinkedList<Segment> segments;
LinkedList<OptimizationStep> optimizationList = new LinkedList<OptimizationStep>();
LinkedList<OptimizationStep> unusedResultsList = new LinkedList<OptimizationStep>();
OptimizationStep stepBest = new OptimizationStep();
// local copy of changesThreshold, that can safely be changed over optimization iterations
float changesThresholdLocal = changesThreshold;
// local copies of prefNumber, absoluteMin and absoluteMax, to make a dependency on track length possible
int prefNumberLocal = prefNumber;
int absoluteMaxLocal = absoluteMax;
int absoluteMinLocal = absoluteMin;
// if the number of segments should depend on the duration of the track, calculate new values for prefNumber,
// absoluteMax and absoluteMin with the duration of the track
if (durationDependent) {
double trackDurationInHours = track.getDuration() / 3600000.0;
prefNumberLocal = (int) Math.round(trackDurationInHours * prefNumberLocal);
absoluteMaxLocal = (int) Math.round(trackDurationInHours * absoluteMax);
absoluteMinLocal = (int) Math.round(trackDurationInHours * absoluteMin);
//make sure prefNumberLocal will never be 0 or negative
if (prefNumberLocal <= 0) {
prefNumberLocal = 1;
}
logger.info("Numbers of segments are set to be relative to track duration. Therefore for {} the preferred "
+ "number of segments is {}", mediaUrl, prefNumberLocal);
}
logger.info("Starting video segmentation of {}", mediaUrl);
// optimization loop to get a segmentation with a number of segments close
// to the desired number of segments
while (!endOptimization) {
mpeg7 = mpeg7CatalogService.newInstance();
videoContent = mpeg7.addVideoContent("videosegment",
contentTime, contentLocator);
// run the segmentation with FFmpeg
segments = runSegmentationFFmpeg(track, videoContent, mediaFile, changesThresholdLocal);
// calculate errors for "normal" and filtered segmentation
// and compare them to find better optimization.
// "normal"
OptimizationStep currentStep = new OptimizationStep(changesThresholdLocal, segments.size(), prefNumberLocal,
mpeg7, segments);
// filtered
LinkedList<Segment> segmentsNew = new LinkedList<Segment>();
OptimizationStep currentStepFiltered = new OptimizationStep(
changesThresholdLocal, 0,
prefNumberLocal, filterSegmentation(segments, track, segmentsNew, stabilityThreshold * 1000), segments);
currentStepFiltered.setSegmentNumAndRecalcErrors(segmentsNew.size());
logger.info("Segmentation yields {} segments after filtering", segmentsNew.size());
OptimizationStep currentStepBest;
// save better optimization in optimizationList
//
// the unfiltered segmentation is better if
// - the error is smaller than the error of the filtered segmentation
// OR - the filtered number of segments is smaller than the preferred number
// - and the unfiltered number of segments is bigger than a value that should roughly estimate how many
// segments with the length of the stability threshold could maximally be in a video
// (this is to make sure that if there are e.g. 1000 segments and the filtering would yield
// smaller and smaller results, the stability threshold won't be optimized in the wrong direction)
// - and the filtered segmentation is not already better than the maximum error
if (currentStep.getErrorAbs() <= currentStepFiltered.getErrorAbs() || (segmentsNew.size() < prefNumberLocal
&& currentStep.getSegmentNum() > (track.getDuration() / 1000.0f) / (stabilityThreshold / 2)
&& !(currentStepFiltered.getErrorAbs() <= maxError))) {
optimizationList.add(currentStep);
Collections.sort(optimizationList);
currentStepBest = currentStep;
unusedResultsList.add(currentStepFiltered);
} else {
optimizationList.add(currentStepFiltered);
Collections.sort(optimizationList);
currentStepBest = currentStepFiltered;
}
cycleCount++;
logger.debug("errorAbs = {}, error = {}", currentStep.getErrorAbs(), currentStep.getError());
logger.debug("changesThreshold = {}", changesThresholdLocal);
logger.debug("cycleCount = {}", cycleCount);
// end optimization if maximum number of cycles is reached or if the segmentation is good enough
if (cycleCount >= maxCycles || currentStepBest.getErrorAbs() <= maxError) {
endOptimization = true;
if (optimizationList.size() > 0) {
if (optimizationList.getFirst().getErrorAbs() <= optimizationList.getLast().getErrorAbs()
&& optimizationList.getFirst().getError() >= 0) {
stepBest = optimizationList.getFirst();
} else {
stepBest = optimizationList.getLast();
}
}
// just to be sure, check if one of the unused results was better
for (OptimizationStep currentUnusedStep : unusedResultsList) {
if (currentUnusedStep.getErrorAbs() < stepBest.getErrorAbs()) {
stepBest = unusedResultsList.getFirst();
}
}
// continue optimization, calculate new changes threshold for next iteration of optimization
} else {
OptimizationStep first = optimizationList.getFirst();
OptimizationStep last = optimizationList.getLast();
// if this was the first iteration or there are only positive or negative errors,
// estimate a new changesThreshold based on the one yielding the smallest error
if (optimizationList.size() == 1 || first.getError() < 0 || last.getError() > 0) {
if (currentStepBest.getError() >= 0) {
// if the error is smaller or equal to 1, increase changes threshold weighted with the error
if (currentStepBest.getError() <= 1) {
changesThresholdLocal += changesThresholdLocal * currentStepBest.getError();
} else {
// if there are more than 2000 segments in the first iteration, set changes threshold to 0.2
// to faster reach reasonable segment numbers
if (cycleCount <= 1 && currentStep.getSegmentNum() > 2000) {
changesThresholdLocal = 0.2f;
// if the error is bigger than one, double the changes threshold, because multiplying
// with a large error can yield a much too high changes threshold
} else {
changesThresholdLocal *= 2;
}
}
} else {
changesThresholdLocal /= 2;
}
logger.debug("onesided optimization yields new changesThreshold = {}", changesThresholdLocal);
// if there are already iterations with positive and negative errors, choose a changesThreshold between those
} else {
// for simplicity a linear relationship between the changesThreshold
// and the number of generated segments is assumed and based on that
// the expected correct changesThreshold is calculated
// the new changesThreshold is calculated by averaging the the mean and the mean weighted with errors
// because this seemed to yield better results in several cases
float x = (first.getSegmentNum() - prefNumberLocal) / (float) (first.getSegmentNum() - last.getSegmentNum());
float newX = ((x + 0.5f) * 0.5f);
changesThresholdLocal = first.getChangesThreshold() * (1 - newX) + last.getChangesThreshold() * newX;
logger.debug("doublesided optimization yields new changesThreshold = {}", changesThresholdLocal);
}
}
}
// after optimization of the changes threshold, the minimum duration for a segment
// (stability threshold) is optimized if the result is still not good enough
int threshLow = stabilityThreshold * 1000;
int threshHigh = threshLow + (threshLow / 2);
LinkedList<Segment> tmpSegments;
float smallestError = Float.MAX_VALUE;
int bestI = threshLow;
segments = stepBest.getSegments();
// if the error is negative (which means there are already too few segments) or if the error
// is smaller than the maximum error, the stability threshold will not be optimized
if (stepBest.getError() <= maxError) {
threshHigh = stabilityThreshold * 1000;
}
for (int i = threshLow; i <= threshHigh; i = i + 1000) {
tmpSegments = new LinkedList<Segment>();
filterSegmentation(segments, track, tmpSegments, i);
float newError = OptimizationStep.calculateErrorAbs(tmpSegments.size(), prefNumberLocal);
if (newError < smallestError) {
smallestError = newError;
bestI = i;
}
}
tmpSegments = new LinkedList<Segment>();
mpeg7 = filterSegmentation(segments, track, tmpSegments, bestI);
// for debugging: output of final segmentation after optimization
logger.debug("result segments:");
for (int i = 0; i < tmpSegments.size(); i++) {
int[] tmpLog2 = new int[7];
tmpLog2[0] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getHour();
tmpLog2[1] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getMinutes();
tmpLog2[2] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getSeconds();
tmpLog2[3] = tmpSegments.get(i).getMediaTime().getMediaDuration().getHours();
tmpLog2[4] = tmpSegments.get(i).getMediaTime().getMediaDuration().getMinutes();
tmpLog2[5] = tmpSegments.get(i).getMediaTime().getMediaDuration().getSeconds();
Object[] tmpLog1 = {tmpLog2[0], tmpLog2[1], tmpLog2[2], tmpLog2[3], tmpLog2[4], tmpLog2[5], tmpLog2[6]};
tmpLog1[6] = tmpSegments.get(i).getIdentifier();
logger.debug("s:{}:{}:{}, d:{}:{}:{}, {}", tmpLog1);
}
logger.info("Optimized Segmentation yields (after {} iteration" + (cycleCount == 1 ? "" : "s") + ") {} segments",
cycleCount, tmpSegments.size());
// if no reasonable segmentation could be found, instead return a uniform segmentation
if (tmpSegments.size() < absoluteMinLocal || tmpSegments.size() > absoluteMaxLocal) {
mpeg7 = uniformSegmentation(track, tmpSegments, prefNumberLocal);
logger.info("Since no reasonable segmentation could be found, a uniform segmentation was created");
}
return mpeg7;
}
/**
* Does the actual segmentation with an FFmpeg call, adds the segments to the given videoContent of a catalog and
* returns a list with the resulting segments
*
* @param track the element to analyze
* @param videoContent the videoContent of the Mpeg7Catalog that the segments should be added to
* @param mediaFile the file of the track to analyze
* @param changesThreshold the changesThreshold that is used as option for the FFmpeg call
* @return a list of the resulting segments
* @throws IOException
* @throws VideoSegmenterException
*/
private LinkedList<Segment> runSegmentationFFmpeg(Track track, Video videoContent, File mediaFile,
float changesThreshold) throws IOException, VideoSegmenterException {
String[] command = new String[] {
binary,
"-nostats", "-nostdin",
"-i", mediaFile.getAbsolutePath(),
"-filter:v", "select=gt(scene\\," + changesThreshold + "),showinfo",
"-f", "null",
"-"
};
logger.info("Detecting video segments using command: {}", (Object) command);
ProcessBuilder pbuilder = new ProcessBuilder(command);
List<String> segmentsStrings = new LinkedList<>();
Process process = pbuilder.start();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
String line = reader.readLine();
while (null != line) {
if (line.startsWith("[Parsed_showinfo")) {
segmentsStrings.add(line);
}
line = reader.readLine();
}
} catch (IOException e) {
logger.error("Error executing ffmpeg: {}", e.getMessage());
}
// [Parsed_showinfo_1 @ 0x157fb40] n:0 pts:12 pts_time:12 pos:227495
// fmt:rgb24 sar:0/1 s:320x240 i:P iskey:1 type:I checksum:8DF39EA9
// plane_checksum:[8DF39EA9]
int segmentcount = 1;
LinkedList<Segment> segments = new LinkedList<>();
if (segmentsStrings.size() == 0) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
segments.add(s);
} else {
long starttime = 0;
long endtime = 0;
Pattern pattern = Pattern.compile("pts_time\\:\\d+(\\.\\d+)?");
for (String seginfo : segmentsStrings) {
Matcher matcher = pattern.matcher(seginfo);
String time = "";
while (matcher.find()) {
time = matcher.group().substring(9);
}
if ("".equals(time)) {
// continue if the showinfo does not contain any time information. This may happen since the FFmpeg showinfo
// filter is used for multiple purposes.
continue;
}
try {
endtime = Math.round(Float.parseFloat(time) * 1000);
} catch (NumberFormatException e) {
logger.error("Unable to parse FFmpeg output, likely FFmpeg version mismatch!", e);
throw new VideoSegmenterException(e);
}
long segmentLength = endtime - starttime;
if (1000 * stabilityThresholdPrefilter < segmentLength) {
Segment segment = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount);
segment.setMediaTime(new MediaRelTimeImpl(starttime,
endtime - starttime));
logger.debug("Created segment {} at start time {} with duration {}", segmentcount, starttime, endtime);
segments.add(segment);
segmentcount++;
starttime = endtime;
}
}
// Add last segment
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(starttime, track.getDuration() - starttime));
logger.debug("Created segment {} at start time {} with duration {}", segmentcount, starttime,
track.getDuration() - endtime);
segments.add(s);
}
logger.info("Segmentation of {} yields {} segments",
mediaFile.toURI().toURL(), segments.size());
return segments;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#process(org.opencastproject.job.api.Job)
*/
@Override
protected String process(Job job) throws Exception {
Operation op = null;
String operation = job.getOperation();
List<String> arguments = job.getArguments();
try {
op = Operation.valueOf(operation);
switch (op) {
case Segment:
Track track = (Track) MediaPackageElementParser
.getFromXml(arguments.get(0));
Catalog catalog = segment(job, track);
return MediaPackageElementParser.getAsXml(catalog);
default:
throw new IllegalStateException(
"Don't know how to handle operation '" + operation
+ "'");
}
} catch (IllegalArgumentException e) {
throw new ServiceRegistryException(
"This service can't handle operations of type '" + op + "'",
e);
} catch (IndexOutOfBoundsException e) {
throw new ServiceRegistryException(
"This argument list for operation '" + op
+ "' does not meet expectations", e);
} catch (Exception e) {
throw new ServiceRegistryException("Error handling operation '"
+ op + "'", e);
}
}
/**
* Merges small subsequent segments (with high difference) into a bigger one
*
* @param segments list of segments to be filtered
* @param track the track that is segmented
* @param segmentsNew will be set to list of new segments (pass null if not required)
* @return Mpeg7Catalog that can later be saved in a Catalog as endresult
*/
protected Mpeg7Catalog filterSegmentation(
LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew) {
int mergeThresh = stabilityThreshold * 1000;
return filterSegmentation(segments, track, segmentsNew, mergeThresh);
}
/**
* Merges small subsequent segments (with high difference) into a bigger one
*
* @param segments list of segments to be filtered
* @param track the track that is segmented
* @param segmentsNew will be set to list of new segments (pass null if not required)
* @param mergeThresh minimum duration for a segment in milliseconds
* @return Mpeg7Catalog that can later be saved in a Catalog as endresult
*/
protected Mpeg7Catalog filterSegmentation(
LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew, int mergeThresh) {
if (segmentsNew == null) {
segmentsNew = new LinkedList<Segment>();
}
boolean merging = false;
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
int segmentcount = 1;
MediaTimePoint currentSegStart = new MediaTimePointImpl();
for (Segment o : segments) {
// if the current segment is shorter than merge treshold start merging
if (o.getMediaTime().getMediaDuration().getDurationInMilliseconds() <= mergeThresh) {
// start merging and save beginning of new segment that will be generated
if (!merging) {
currentSegStart = o.getMediaTime().getMediaTimePoint();
merging = true;
}
// current segment is longer than merge threshold
} else {
long currentSegDuration = o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
long currentSegEnd = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds()
+ currentSegDuration;
if (merging) {
long newDuration = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds()
- currentSegStart.getTimeInMilliseconds();
// if new segment would be long enough
// save new segment that merges all previously skipped short segments
if (newDuration >= mergeThresh) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
segmentsNew.add(s);
// copy the following long segment to new list
Segment s2 = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount++);
s2.setMediaTime(o.getMediaTime());
segmentsNew.add(s2);
// if too short split new segment in middle and merge halves to
// previous and following segments
} else {
long followingStartOld = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
long newSplit = (currentSegStart.getTimeInMilliseconds() + followingStartOld) / 2;
long followingEnd = followingStartOld + o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
long followingDuration = followingEnd - newSplit;
// if at beginning, don't split, just merge to first large segment
if (segmentsNew.isEmpty()) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(0, followingEnd));
segmentsNew.add(s);
} else {
long previousStart = segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
// adjust end time of previous segment to split time
segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(previousStart, newSplit - previousStart));
// create new segment starting at split time
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(newSplit, followingDuration));
segmentsNew.add(s);
}
}
merging = false;
// copy segments that are long enough to new list (with corrected number)
} else {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount++);
s.setMediaTime(o.getMediaTime());
segmentsNew.add(s);
}
}
}
// if there is an unfinished merging process after going through all segments
if (merging && !segmentsNew.isEmpty()) {
long newDuration = track.getDuration() - currentSegStart.getTimeInMilliseconds();
// if merged segment is long enough, create new segment
if (newDuration >= mergeThresh) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
segmentsNew.add(s);
// if not long enough, merge with previous segment
} else {
newDuration = track.getDuration() - segmentsNew.getLast().getMediaTime().getMediaTimePoint()
.getTimeInMilliseconds();
segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(segmentsNew.getLast().getMediaTime()
.getMediaTimePoint().getTimeInMilliseconds(), newDuration));
}
}
// if there is no segment in the list (to merge with), create new
// segment spanning the whole video
if (segmentsNew.isEmpty()) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
segmentsNew.add(s);
}
return mpeg7;
}
/**
* Creates a uniform segmentation for a given track, with prefNumber as the number of segments
* which will all have the same length
*
* @param track the track that is segmented
* @param segmentsNew will be set to list of new segments (pass null if not required)
* @param prefNumber number of generated segments
* @return Mpeg7Catalog that can later be saved in a Catalog as endresult
*/
protected Mpeg7Catalog uniformSegmentation(Track track, LinkedList<Segment> segmentsNew, int prefNumber) {
if (segmentsNew == null) {
segmentsNew = new LinkedList<Segment>();
}
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
long segmentDuration = track.getDuration() / prefNumber;
long currentSegStart = 0;
// create "prefNumber"-many segments that all have the same length
for (int i = 1; i < prefNumber; i++) {
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + i);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart, segmentDuration));
segmentsNew.add(s);
currentSegStart += segmentDuration;
}
// add last segment separately to make sure the last segment ends exactly at the end of the track
Segment s = videoContent.getTemporalDecomposition()
.createSegment("segment-" + prefNumber);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart, track.getDuration() - currentSegStart));
segmentsNew.add(s);
return mpeg7;
}
/**
* Sets the workspace
*
* @param workspace
* an instance of the workspace
*/
@Reference
protected void setWorkspace(Workspace workspace) {
this.workspace = workspace;
}
/**
* Sets the mpeg7CatalogService
*
* @param mpeg7CatalogService
* an instance of the mpeg7 catalog service
*/
@Reference(name = "Mpeg7Service")
protected void setMpeg7CatalogService(
Mpeg7CatalogService mpeg7CatalogService) {
this.mpeg7CatalogService = mpeg7CatalogService;
}
/**
* Sets the receipt service
*
* @param serviceRegistry
* the service registry
*/
@Reference
protected void setServiceRegistry(ServiceRegistry serviceRegistry) {
this.serviceRegistry = serviceRegistry;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getServiceRegistry()
*/
@Override
protected ServiceRegistry getServiceRegistry() {
return serviceRegistry;
}
/**
* Callback for setting the security service.
*
* @param securityService
* the securityService to set
*/
@Reference
public void setSecurityService(SecurityService securityService) {
this.securityService = securityService;
}
/**
* Callback for setting the user directory service.
*
* @param userDirectoryService
* the userDirectoryService to set
*/
@Reference
public void setUserDirectoryService(
UserDirectoryService userDirectoryService) {
this.userDirectoryService = userDirectoryService;
}
/**
* Sets a reference to the organization directory service.
*
* @param organizationDirectory
* the organization directory
*/
@Reference
public void setOrganizationDirectoryService(
OrganizationDirectoryService organizationDirectory) {
this.organizationDirectoryService = organizationDirectory;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getSecurityService()
*/
@Override
protected SecurityService getSecurityService() {
return securityService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getUserDirectoryService()
*/
@Override
protected UserDirectoryService getUserDirectoryService() {
return userDirectoryService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getOrganizationDirectoryService()
*/
@Override
protected OrganizationDirectoryService getOrganizationDirectoryService() {
return organizationDirectoryService;
}
}