ExecuteServiceImpl.java
/*
* Licensed to The Apereo Foundation under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
*
* The Apereo Foundation licenses this file to you under the Educational
* Community License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License
* at:
*
* http://opensource.org/licenses/ecl2.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package org.opencastproject.execute.impl;
import org.opencastproject.execute.api.ExecuteException;
import org.opencastproject.execute.api.ExecuteService;
import org.opencastproject.job.api.AbstractJobProducer;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.MediaPackage;
import org.opencastproject.mediapackage.MediaPackageElement;
import org.opencastproject.mediapackage.MediaPackageElement.Type;
import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
import org.opencastproject.mediapackage.MediaPackageElementFlavor;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageException;
import org.opencastproject.mediapackage.MediaPackageParser;
import org.opencastproject.mediapackage.UnsupportedElementException;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.ConfigurationException;
import org.opencastproject.util.IoSupport;
import org.opencastproject.util.LoadUtil;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.workspace.api.Workspace;
import org.apache.commons.lang3.StringUtils;
import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ManagedService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Dictionary;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Implements a service that runs CLI commands with MediaPackage elements as arguments
*/
@Component(
immediate = true,
service = { ExecuteService.class,ManagedService.class },
property = {
"service.description=Execute Service",
"service.pid=org.opencastproject.execute.impl.ExecuteServiceImpl"
}
)
public class ExecuteServiceImpl extends AbstractJobProducer implements ExecuteService, ManagedService {
public enum Operation {
Execute_Element, Execute_Mediapackage
}
/** The logging facility */
private static final Logger logger = LoggerFactory.getLogger(ExecuteServiceImpl.class);
/** Reference to the receipt service */
private ServiceRegistry serviceRegistry = null;
/** The security service */
protected SecurityService securityService = null;
/** The user directory service */
protected UserDirectoryService userDirectoryService = null;
/** The organization directory service */
protected OrganizationDirectoryService organizationDirectoryService = null;
/** The workspace service */
protected Workspace workspace;
/**
* List of allowed commands that can be run with an executor. By convention, an empty set doesn't mean any command can
* be run. An '*' in the service configuration means any command can be executed
*/
protected final Set<String> allowedCommands = new HashSet<String>();
/** Bundle property specifying which commands can be run with this executor */
public static final String COMMANDS_ALLOWED_PROPERTY = "commands.allowed";
/** To allow command-line parameter substitutions configured globally i.e. in config.properties */
private BundleContext bundleContext;
/** To allow command-line parameter substitutions configured at the service level */
@SuppressWarnings("rawtypes")
private Dictionary properties = null;
/** The approximate load placed on the system by running an execute operation */
public static final float DEFAULT_EXECUTE_JOB_LOAD = 0.1f;
/** The key to look for in the service configuration file to override the {@link DEFAULT_EXECUTE_JOB_LOAD} */
public static final String EXECUTE_JOB_LOAD_KEY = "job.load.execute";
private float executeJobLoad = 1.0f;
/**
* Creates a new instance of the execute service.
*/
public ExecuteServiceImpl() {
super(JOB_TYPE);
}
/**
* Activates this component with its properties once all of the collaborating services have been set
*
* @param cc
* The component's context, containing the properties used for configuration
*/
@Override
@Activate
public void activate(ComponentContext cc) {
super.activate(cc);
properties = cc.getProperties();
if (properties != null) {
String commandString = (String) properties.get(COMMANDS_ALLOWED_PROPERTY);
if (StringUtils.isNotBlank(commandString)) {
logger.info("Execute Service permitted commands: {}", commandString);
for (String command : commandString.split("\\s+"))
allowedCommands.add(command);
}
}
this.bundleContext = cc.getBundleContext();
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
* org.opencastproject.mediapackage.MediaPackageElement, java.lang.String,
* org.opencastproject.mediapackage.MediaPackageElement.Type, float)
* @throws IllegalArgumentException
* if the input arguments are incorrect
* @throws ExecuteException
* if an internal error occurs
*/
@Override
public Job execute(String exec, String params, MediaPackageElement inElement, String outFileName, Type expectedType,
float load) throws ExecuteException, IllegalArgumentException {
logger.debug("Creating Execute Job for command: {}", exec);
if (StringUtils.isBlank(exec))
throw new IllegalArgumentException("The command to execute cannot be null");
if (StringUtils.isBlank(params))
throw new IllegalArgumentException("The command arguments cannot be null");
if (inElement == null)
throw new IllegalArgumentException("The input MediaPackage element cannot be null");
outFileName = StringUtils.trimToNull(outFileName);
if ((outFileName == null) && (expectedType != null) || (outFileName != null) && (expectedType == null))
throw new IllegalArgumentException("Expected element type and output filename cannot be null");
try {
List<String> paramList = new ArrayList<String>(5);
paramList.add(exec);
paramList.add(params);
paramList.add(MediaPackageElementParser.getAsXml(inElement));
paramList.add(outFileName);
paramList.add((expectedType == null) ? null : expectedType.toString());
return serviceRegistry.createJob(JOB_TYPE, Operation.Execute_Element.toString(), paramList, load);
} catch (ServiceRegistryException e) {
throw new ExecuteException(String.format("Unable to create a job of type '%s'", JOB_TYPE), e);
} catch (MediaPackageException e) {
throw new ExecuteException("Error serializing an element", e);
}
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
* org.opencastproject.mediapackage.MediaPackage, java.lang.String,
* org.opencastproject.mediapackage.MediaPackageElement.Type, float)
*/
@Override
public Job execute(String exec, String params, MediaPackage mp, String outFileName, Type expectedType, float load)
throws ExecuteException {
if (StringUtils.isBlank(exec))
throw new IllegalArgumentException("The command to execute cannot be null");
if (StringUtils.isBlank(params))
throw new IllegalArgumentException("The command arguments cannot be null");
if (mp == null)
throw new IllegalArgumentException("The input MediaPackage cannot be null");
outFileName = StringUtils.trimToNull(outFileName);
if ((outFileName == null) && (expectedType != null) || (outFileName != null) && (expectedType == null))
throw new IllegalArgumentException("Expected element type and output filename cannot be null");
try {
List<String> paramList = new ArrayList<String>(5);
paramList.add(exec);
paramList.add(params);
paramList.add(MediaPackageParser.getAsXml(mp));
paramList.add(outFileName);
paramList.add((expectedType == null) ? null : expectedType.toString());
return serviceRegistry.createJob(JOB_TYPE, Operation.Execute_Mediapackage.toString(), paramList, load);
} catch (ServiceRegistryException e) {
throw new ExecuteException(String.format("Unable to create a job of type '%s'", JOB_TYPE), e);
}
}
/**
* {@inheritDoc}
*
* @throws ExecuteException
*
* @see org.opencastproject.job.api.AbstractJobProducer#process(org.opencastproject.job.api.Job)
*/
@Override
protected String process(Job job) throws ExecuteException {
List<String> arguments = new ArrayList<String>(job.getArguments());
// Check this operation is allowed
if (!allowedCommands.contains("*") && !allowedCommands.contains(arguments.get(0)))
throw new ExecuteException("Command '" + arguments.get(0) + "' is not allowed");
String outFileName = null;
String strAux = null;
MediaPackage mp = null;
Type expectedType = null;
MediaPackageElement element = null;
Operation op = null;
try {
op = Operation.valueOf(job.getOperation());
int nargs = arguments.size();
if (nargs != 3 && nargs != 5) {
throw new IndexOutOfBoundsException(
"Incorrect number of parameters for operation execute_" + op + ": " + arguments.size());
}
if (nargs == 5) {
strAux = arguments.remove(4);
expectedType = (strAux == null) ? null : Type.valueOf(strAux);
outFileName = StringUtils.trimToNull(arguments.remove(3));
if ((StringUtils.isNotBlank(outFileName) && (expectedType == null))
|| (StringUtils.isBlank(outFileName) && (expectedType != null))) {
throw new ExecuteException("The output type and filename must be both specified");
}
outFileName = (outFileName == null) ? null : job.getId() + "_" + outFileName;
}
switch (op) {
case Execute_Mediapackage:
mp = MediaPackageParser.getFromXml(arguments.remove(2));
return doProcess(arguments, mp, outFileName, expectedType);
case Execute_Element:
element = MediaPackageElementParser.getFromXml(arguments.remove(2));
return doProcess(arguments, element, outFileName, expectedType);
default:
throw new IllegalStateException("Don't know how to handle operation '" + job.getOperation() + "'");
}
} catch (MediaPackageException e) {
throw new ExecuteException("Error unmarshalling the input mediapackage/element", e);
} catch (IllegalArgumentException e) {
throw new ExecuteException("This service can't handle operations of type '" + op + "'", e);
} catch (IndexOutOfBoundsException e) {
throw new ExecuteException("The argument list for operation '" + op + "' does not meet expectations", e);
}
}
/**
* Does the actual processing, given a mediapackage (Execute Once WOH)
*
* @param arguments
* The list containing the program and its arguments
* @param mp
* MediaPackage used in the operation
* @param outFileName
* The name of the resulting file
* @param expectedType
* The expected element type
* @return A {@code String} containing the command output
* @throws ExecuteException
* if some internal error occurred
*/
protected String doProcess(List<String> arguments, MediaPackage mp, String outFileName, Type expectedType)
throws ExecuteException {
String params = arguments.remove(1);
File outFile = null;
MediaPackageElement[] elements = null;
try {
if (outFileName != null) {
// FIXME : Find a better way to place the output File
File firstElement = workspace.get(mp.getElements()[0].getURI());
outFile = new File(firstElement.getParentFile(), outFileName);
}
// Get the substitution pattern.
// The following pattern matches, any construct with the form
// #{name}
// , where 'name' is the value of a certain property. It is stored in the backreference group 1.
// Optionally, expressions can take a parameter, like
// #{name(parameter)}
// , where 'parameter' is the name of a certain parameter.
// If specified, 'parameter' is stored in the group 2. Otherwise it's null.
// Both name and parameter match any character sequence that does not contain {, }, ( or ) .
Pattern pat = Pattern.compile("#\\{([^\\{\\}\\(\\)]+)(?:\\(([^\\{\\}\\(\\)]+)\\))?\\}");
// Substitute the appearances of the patterns with the actual absolute paths
Matcher matcher = pat.matcher(params);
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
// group(1) = property. group(2) = (optional) parameter
if (matcher.group(1).equals("id")) {
matcher.appendReplacement(sb, mp.getIdentifier().toString());
} else if (matcher.group(1).equals("flavor")) {
elements = mp.getElementsByFlavor(MediaPackageElementFlavor.parseFlavor(matcher.group(2)));
if (elements.length == 0)
throw new ExecuteException("No elements in the MediaPackage match the flavor '" + matcher.group(2) + "'.");
if (elements.length > 1)
logger.warn("Found more than one element with flavor '{}'. Using {} by default...", matcher.group(2),
elements[0].getIdentifier());
File elementFile = workspace.get(elements[0].getURI());
matcher.appendReplacement(sb, elementFile.getAbsolutePath());
} else if (matcher.group(1).equals("tags")) {
elements = mp.getElementsByTags(Arrays.asList(StringUtils.split(matcher.group(2), ",")));
if (elements.length == 0)
throw new ExecuteException("No elements in the MediaPackage match the tags '" + matcher.group(2) + "'.");
if (elements.length > 1)
logger.warn("Found more than one element with matching tags '{}'. Using {} by default...", matcher.group(2),
elements[0].getIdentifier());
File elementFile = workspace.get(elements[0].getURI());
matcher.appendReplacement(sb, elementFile.getAbsolutePath());
} else if (matcher.group(1).equals("out")) {
matcher.appendReplacement(sb, outFile.getAbsolutePath());
} else if (matcher.group(1).equals("org_id")) {
matcher.appendReplacement(sb, securityService.getOrganization().getId());
} else if (properties.get(matcher.group(1)) != null) {
matcher.appendReplacement(sb, (String) properties.get(matcher.group(1)));
} else if (bundleContext.getProperty(matcher.group(1)) != null) {
matcher.appendReplacement(sb, bundleContext.getProperty(matcher.group(1)));
}
}
matcher.appendTail(sb);
params = sb.toString();
} catch (IllegalArgumentException e) {
throw new ExecuteException("Tag 'flavor' must specify a valid MediaPackage element flavor.", e);
} catch (NotFoundException e) {
throw new ExecuteException(
"The element '" + elements[0].getURI().toString() + "' does not exist in the workspace.", e);
} catch (IOException e) {
throw new ExecuteException("Error retrieving MediaPackage element from workspace: '"
+ elements[0].getURI().toString() + "'.", e);
}
arguments.addAll(splitParameters(params));
return runCommand(arguments, outFile, expectedType);
}
/**
* Does the actual processing, given a mediapackage element (Execute Many WOH)
*
* @param arguments
* The list containing the program and its arguments
* @param outFileName
* The name of the resulting file
* @param expectedType
* The expected element type
* @return A {@code String} containing the command output
* @throws ExecuteException
* if some internal error occurred
*/
protected String doProcess(List<String> arguments, MediaPackageElement element, String outFileName, Type expectedType)
throws ExecuteException {
// arguments(1) contains a list of space-separated arguments for the command
String params = arguments.remove(1);
arguments.addAll(splitParameters(params));
File outFile = null;
try {
// Get the track file from the workspace
File trackFile = workspace.get(element.getURI());
// Put the destination file in the same folder as the source file
if (outFileName != null)
outFile = new File(trackFile.getParentFile(), outFileName);
// Substitute the appearances of the patterns with the actual absolute paths
for (int i = 1; i < arguments.size(); i++) {
if (arguments.get(i).contains(INPUT_FILE_PATTERN)) {
arguments.set(i, arguments.get(i).replace(INPUT_FILE_PATTERN, trackFile.getAbsolutePath()));
continue;
}
if (arguments.get(i).contains(OUTPUT_FILE_PATTERN)) {
if (outFile != null) {
arguments.set(i, arguments.get(i).replace(OUTPUT_FILE_PATTERN, outFile.getAbsolutePath()));
continue;
} else {
logger.error("{} pattern found, but no valid output filename was specified", OUTPUT_FILE_PATTERN);
throw new ExecuteException(
OUTPUT_FILE_PATTERN + " pattern found, but no valid output filename was specified");
}
}
if (arguments.get(i).contains(MP_ID_PATTERN)) {
arguments.set(i, arguments.get(i).replace(MP_ID_PATTERN, element.getMediaPackage().getIdentifier().toString()));
}
if (arguments.get(i).contains(ORG_ID_PATTERN)) {
arguments.set(i, arguments.get(i).replace(ORG_ID_PATTERN, securityService.getOrganization().getId()));
}
}
return runCommand(arguments, outFile, expectedType);
} catch (IOException e) {
logger.error("Error retrieving file from workspace: {}", element.getURI());
throw new ExecuteException("Error retrieving file from workspace: " + element.getURI(), e);
} catch (NotFoundException e) {
logger.error("Element '{}' cannot be found in the workspace.", element.getURI());
throw new ExecuteException("Element " + element.getURI() + " cannot be found in the workspace");
}
}
private String runCommand(List<String> command, File outFile, Type expectedType) throws ExecuteException {
Process p = null;
int result = 0;
try {
logger.info("Running command {}", command.get(0));
logger.debug("Starting subprocess {} with arguments {}", command.get(0),
StringUtils.join(command.subList(1, command.size()), ", "));
ProcessBuilder pb = new ProcessBuilder(command);
pb.redirectErrorStream(true);
p = pb.start();
BufferedReader stdout = new BufferedReader(new InputStreamReader(p.getInputStream()));
String line;
while ((line = stdout.readLine()) != null) {
logger.debug(line);
}
result = p.waitFor();
logger.debug("Command {} finished with result {}", command.get(0), result);
if (result == 0) {
// Read the command output
if (outFile != null) {
if (outFile.isFile()) {
URI newURI = workspace.putInCollection(ExecuteService.COLLECTION, outFile.getName(), new FileInputStream(outFile));
if (outFile.delete()) {
logger.debug("Deleted the local copy of the encoded file at {}", outFile.getAbsolutePath());
} else {
logger.warn("Unable to delete the encoding output at {}", outFile.getAbsolutePath());
}
return MediaPackageElementParser.getAsXml(MediaPackageElementBuilderFactory.newInstance()
.newElementBuilder().elementFromURI(newURI, expectedType, null));
} else {
throw new ExecuteException("Expected output file does not exist: " + outFile.getAbsolutePath());
}
}
return "";
} else {
throw new ExecuteException(String.format("Process %s returned error code %d", command.get(0), result));
}
} catch (InterruptedException e) {
throw new ExecuteException("The executor thread has been unexpectedly interrupted", e);
} catch (IOException e) {
// Only log the first argument, the executable, as other arguments may contain sensitive values
// e.g. MySQL password/user, paths, etc. that should not be shown to caller
logger.error("Could not start subprocess {}", command.get(0));
throw new ExecuteException("Could not start subprocess: " + command.get(0), e);
} catch (UnsupportedElementException e) {
throw new ExecuteException("Couldn't create a new MediaPackage element of type " + expectedType.toString(), e);
} catch (ConfigurationException e) {
throw new ExecuteException("Couldn't instantiate a new MediaPackage element builder", e);
} catch (MediaPackageException e) {
throw new ExecuteException("Couldn't serialize a new Mediapackage element of type " + expectedType.toString(), e);
} finally {
IoSupport.closeQuietly(p);
}
}
/**
* Returns a list of strings broken on whitespace characters except where those whitespace characters are escaped or
* quoted.
*
* @return list of individual arguments
*/
private List<String> splitParameters(String input) {
// This delimiter matches any non-escaped quote
final String quoteDelim = "(?<!\\\\)\"";
// This delimiter matches any number of non-escaped spaces
final String spaceDelim = "((?<!\\\\)\\s)+";
ArrayList<String> parsedInput = new ArrayList<String>();
boolean quoted = false;
for (String token1 : input.split(quoteDelim))
if (quoted) {
parsedInput.add(token1);
quoted = false;
} else {
for (String token2 : token1.split(spaceDelim))
// This ignores empty tokens if quotes are at the beginning or the end of the string
if (!token2.isEmpty())
parsedInput.add(token2);
quoted = true;
}
return parsedInput;
}
/**
* Sets the receipt service
*
* @param serviceRegistry
* the service registry
*/
@Reference
public void setServiceRegistry(ServiceRegistry serviceRegistry) {
this.serviceRegistry = serviceRegistry;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getServiceRegistry()
*/
@Override
protected ServiceRegistry getServiceRegistry() {
return serviceRegistry;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getSecurityService()
*/
@Override
protected SecurityService getSecurityService() {
return securityService;
}
/**
* Callback for setting the security service.
*
* @param securityService
* the securityService to set
*/
@Reference
public void setSecurityService(SecurityService securityService) {
this.securityService = securityService;
}
/**
* Callback for setting the user directory service.
*
* @param userDirectoryService
* the userDirectoryService to set
*/
@Reference
public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
this.userDirectoryService = userDirectoryService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getUserDirectoryService()
*/
@Override
protected UserDirectoryService getUserDirectoryService() {
return userDirectoryService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getOrganizationDirectoryService()
*/
@Override
protected OrganizationDirectoryService getOrganizationDirectoryService() {
return organizationDirectoryService;
}
/**
* Sets a reference to the organization directory service.
*
* @param organizationDirectory
* the organization directory
*/
@Reference
public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectory) {
this.organizationDirectoryService = organizationDirectory;
}
/**
* Sets a reference to the workspace service.
*
* @param workspace
*/
@Reference
public void setWorkspace(Workspace workspace) {
this.workspace = workspace;
}
@Override
public void updated(@SuppressWarnings("rawtypes") Dictionary properties)
throws org.osgi.service.cm.ConfigurationException {
executeJobLoad = LoadUtil.getConfiguredLoadValue(properties, EXECUTE_JOB_LOAD_KEY, DEFAULT_EXECUTE_JOB_LOAD,
serviceRegistry);
}
}