YouTubeV3PublicationServiceImpl.java
/*
* Licensed to The Apereo Foundation under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
*
* The Apereo Foundation licenses this file to you under the Educational
* Community License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License
* at:
*
* http://opensource.org/licenses/ecl2.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package org.opencastproject.publication.youtube;
import org.opencastproject.job.api.AbstractJobProducer;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.Attachment;
import org.opencastproject.mediapackage.MediaPackage;
import org.opencastproject.mediapackage.MediaPackageElement;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageParser;
import org.opencastproject.mediapackage.Publication;
import org.opencastproject.mediapackage.PublicationImpl;
import org.opencastproject.mediapackage.Track;
import org.opencastproject.publication.api.PublicationException;
import org.opencastproject.publication.api.YouTubePublicationService;
import org.opencastproject.publication.youtube.auth.ClientCredentials;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.LoadUtil;
import org.opencastproject.util.MimeTypes;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.util.XProperties;
import org.opencastproject.workspace.api.Workspace;
import com.google.api.services.youtube.model.Playlist;
import com.google.api.services.youtube.model.Video;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.osgi.service.cm.ConfigurationException;
import org.osgi.service.cm.ManagedService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Dictionary;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Publishes media to a YouTube play list.
*/
@Component(
immediate = true,
service = { ManagedService.class,YouTubePublicationService.class },
property = {
"service.description=Publication Service (YouTube API Version 3)"
}
)
public class YouTubeV3PublicationServiceImpl
extends AbstractJobProducer
implements YouTubePublicationService, ManagedService {
/** The load on the system introduced by creating a publish job */
public static final float DEFAULT_YOUTUBE_PUBLISH_JOB_LOAD = 0.1f;
/** The load on the system introduced by creating a retract job */
public static final float DEFAULT_YOUTUBE_RETRACT_JOB_LOAD = 0.1f;
/** The key to look for in the service configuration file to override the {@link #DEFAULT_YOUTUBE_PUBLISH_JOB_LOAD} */
public static final String YOUTUBE_PUBLISH_LOAD_KEY = "job.load.youtube.publish";
/** The key to look for in the service configuration file to override the {@link #DEFAULT_YOUTUBE_RETRACT_JOB_LOAD} */
public static final String YOUTUBE_RETRACT_LOAD_KEY = "job.load.youtube.retract";
public static final String YOUTUBE_ENABLED_KEY = "org.opencastproject.publication.youtube.enabled";
/** The load on the system introduced by creating a publish job */
private float youtubePublishJobLoad = DEFAULT_YOUTUBE_PUBLISH_JOB_LOAD;
/** The load on the system introduced by creating a retract job */
private float youtubeRetractJobLoad = DEFAULT_YOUTUBE_RETRACT_JOB_LOAD;
/** Time to wait between polling for status (milliseconds.) */
private static final long POLL_MILLISECONDS = 30L * 1000L;
/** The channel name */
private static final String CHANNEL_NAME = "youtube";
/** logger instance */
private static final Logger logger = LoggerFactory.getLogger(YouTubeV3PublicationServiceImpl.class);
/** The mime-type of the published element */
private static final String MIME_TYPE = "text/html";
/** List of available operations on jobs */
private enum Operation {
Publish, Retract
}
/** workspace instance */
protected Workspace workspace = null;
/** The remote service registry */
protected ServiceRegistry serviceRegistry = null;
/** The organization directory service */
private OrganizationDirectoryService organizationDirectoryService;
/** The user directory service */
private UserDirectoryService userDirectoryService;
/** The security service */
private SecurityService securityService;
/** YouTube configuration instance */
private final YouTubeAPIVersion3Service youTubeService;
private boolean enabled = false;
private boolean transferAudioLanguage;
private boolean transferMetadataLanguage;
private Map<String, Pattern> languagePatterns;
/**
* The default playlist to publish to, in case there is not enough information in the mediapackage to find a playlist
*/
private String defaultPlaylist;
private Optional<Pattern> ccLicenses;
private boolean makeVideosPrivate;
private YouTubeAPIVersion3Service.PrivacyStatus playlistPrivacy;
private String[] tags;
private XProperties properties = new XProperties();
/**
* The maximum length of a Recording or Series title.
* A value of zero will be treated as no limit
*/
private int maxFieldLength;
/**
* Creates a new instance of the youtube publication service.
*/
YouTubeV3PublicationServiceImpl(final YouTubeAPIVersion3Service youTubeService) throws Exception {
super(JOB_TYPE);
this.youTubeService = youTubeService;
}
/**
* Creates a new instance of the youtube publication service.
*/
public YouTubeV3PublicationServiceImpl() throws Exception {
this(new YouTubeAPIVersion3ServiceImpl());
}
/**
* Called when service activates. Defined in OSGi resource file.
*/
@Override
@Activate
public synchronized void activate(final ComponentContext cc) {
super.activate(cc);
properties.setBundleContext(cc.getBundleContext());
logger.debug("Activated YouTube Publication Service");
}
@Override
public void updated(final Dictionary props) throws ConfigurationException {
properties.merge(props);
enabled = Boolean.valueOf((String) properties.get(YOUTUBE_ENABLED_KEY));
final String dataStore = YouTubeUtils.get(properties, YouTubeKey.credentialDatastore);
try {
if (enabled) {
final ClientCredentials clientCredentials = new ClientCredentials();
clientCredentials.setCredentialDatastore(dataStore);
final String path = YouTubeUtils.get(properties, YouTubeKey.clientSecretsV3);
File secretsFile = new File(path);
if (secretsFile.exists() && !secretsFile.isDirectory()) {
clientCredentials.setClientSecrets(secretsFile);
clientCredentials.setDataStoreDirectory(YouTubeUtils.get(properties, YouTubeKey.dataStore));
//
youTubeService.initialize(clientCredentials);
//
String languageTarget = YouTubeUtils.get(properties, YouTubeKey.languageTarget, false);
transferAudioLanguage = "audio".equals(languageTarget) || "both".equals(languageTarget);
transferMetadataLanguage = "metadata".equals(languageTarget) || "both".equals(languageTarget);
languagePatterns = YouTubeUtils.getAll(properties, YouTubeKey.languagePatterns).entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
e -> Pattern.compile(e.getValue())));
ccLicenses = Optional.ofNullable(
YouTubeUtils.get(properties, YouTubeKey.ccLicenses, false))
.map(Pattern::compile);
tags = StringUtils.split(YouTubeUtils.get(properties, YouTubeKey.keywords), ',');
defaultPlaylist = YouTubeUtils.get(properties, YouTubeKey.defaultPlaylist);
makeVideosPrivate = Strings.CI
.contains(YouTubeUtils.get(properties, YouTubeKey.makeVideosPrivate), "true");
playlistPrivacy = YouTubeAPIVersion3Service.PrivacyStatus.valueOf(Objects.requireNonNullElse(
YouTubeUtils.get(properties, YouTubeKey.playlistPrivacy, false), "public").toUpperCase());
defaultMaxFieldLength(YouTubeUtils.get(properties, YouTubeKey.maxFieldLength, false));
} else {
logger.warn("Client information file does not exist: " + path);
}
} else {
logger.info("YouTube v3 publication service is disabled");
}
} catch (final Exception e) {
throw new ConfigurationException("Failed to load YouTube v3 properties", dataStore, e);
}
youtubePublishJobLoad = LoadUtil.getConfiguredLoadValue(
properties, YOUTUBE_PUBLISH_LOAD_KEY, DEFAULT_YOUTUBE_PUBLISH_JOB_LOAD, serviceRegistry);
youtubeRetractJobLoad = LoadUtil.getConfiguredLoadValue(
properties, YOUTUBE_RETRACT_LOAD_KEY, DEFAULT_YOUTUBE_RETRACT_JOB_LOAD, serviceRegistry);
}
@Override
public Job publish(final MediaPackage mediaPackage, final Track track, final Attachment thumbnail,
final List<Track> captions)
throws PublicationException {
if (mediaPackage.contains(track)) {
try {
final List<String> args = new ArrayList<>();
args.add(MediaPackageParser.getAsXml(mediaPackage));
args.add(track.getIdentifier());
// The thumbnail ID is positional, so keep its slot even when there is no thumbnail
args.add(thumbnail != null ? thumbnail.getIdentifier() : "");
if (captions != null && !captions.isEmpty()) {
final List<String> captionIds = new ArrayList<>();
for (Track caption : captions) {
captionIds.add(caption.getIdentifier());
}
args.add(StringUtils.join(captionIds, ","));
}
return serviceRegistry.createJob(JOB_TYPE, Operation.Publish.toString(), args, youtubePublishJobLoad);
} catch (ServiceRegistryException e) {
throw new PublicationException("Unable to create a job for track: " + track.toString(), e);
}
} else {
throw new IllegalArgumentException("Mediapackage does not contain track " + track.getIdentifier());
}
}
/**
* Publishes the element to the publication channel and returns a reference to the published version of the element.
*
* @param job
* the associated job
* @param mediaPackage
* the mediapackage
* @param elementId
* the mediapackage element id to publish
* @param thumbnailId
* the optional mediapackage thumbnail element id to publish
* @param captions
* the mediapackage caption elements to publish
* @return the published element
* @throws PublicationException
* if publication fails
*/
private Publication publish(final Job job, final MediaPackage mediaPackage, final String elementId,
final String thumbnailId, final List<Track> captions) throws PublicationException {
if (mediaPackage == null) {
throw new IllegalArgumentException("Mediapackage must be specified");
} else if (elementId == null) {
throw new IllegalArgumentException("Mediapackage ID must be specified");
}
final MediaPackageElement element = mediaPackage.getElementById(elementId);
if (element == null) {
throw new IllegalArgumentException("Mediapackage element must be specified");
}
if (element.getIdentifier() == null) {
throw new IllegalArgumentException("Mediapackage element must have an identifier");
}
if (element.getMimeType().toString().matches("text/xml")) {
throw new IllegalArgumentException("Mediapackage element cannot be XML");
}
try {
// create context strategy for publication
final YouTubePublicationAdapter c = new YouTubePublicationAdapter(mediaPackage, workspace);
final File file = workspace.get(element.getURI());
final String episodeName = c.getEpisodeName();
final String episodeLanguage = c.getEpisodeLanguage();
final String language = episodeLanguage == null ? null : languagePatterns.entrySet().stream()
.filter(e -> e.getValue().matcher(c.getEpisodeLanguage()).matches())
.findAny()
.map(Map.Entry::getKey)
.orElse(null);
final UploadProgressListener operationProgressListener = new UploadProgressListener(mediaPackage, file);
final String privacyStatus = makeVideosPrivate ? "private" : "public";
final String episodeLicense = c.getEpisodeLicense();
final boolean isCreativeCommons = episodeLicense != null
&& ccLicenses.map(pattern -> pattern.matcher(episodeLicense).matches()).orElse(false);
final VideoUpload.License license = isCreativeCommons
? VideoUpload.License.creativeCommon
: VideoUpload.License.youtube;
final VideoUpload videoUpload = new VideoUpload(
truncateTitleToMaxFieldLength(episodeName, false),
c.getEpisodeDescription(),
transferMetadataLanguage ? language : null,
transferAudioLanguage ? language : null,
license, privacyStatus, file, operationProgressListener, tags);
final Video video = youTubeService.addVideoToMyChannel(videoUpload);
if (thumbnailId != null) {
final MediaPackageElement thumbnailElement = mediaPackage.getElementById(thumbnailId);
if (thumbnailElement != null) {
try {
final File thumbnailFile = workspace.get(thumbnailElement.getURI());
youTubeService.setThumbnail(video.getId(), thumbnailFile, thumbnailElement.getMimeType().toString());
} catch (IOException | NotFoundException e) {
// The video is already on YouTube at this point. A missing thumbnail should not fail the whole
// publication, which would leave the video there without a way to retract it.
logger.warn("Failed to set thumbnail {} for video {}: {}", thumbnailElement, video.getId(),
e.getMessage());
}
}
}
final int timeoutMinutes = 60;
final long startUploadMilliseconds = new Date().getTime();
while (!operationProgressListener.isComplete()) {
Thread.sleep(POLL_MILLISECONDS);
final long howLongWaitingMinutes = (new Date().getTime() - startUploadMilliseconds) / 60000;
if (howLongWaitingMinutes > timeoutMinutes) {
throw new PublicationException(
"Upload to YouTube exceeded " + timeoutMinutes + " minutes for episode " + episodeName);
}
}
String playlistName = StringUtils.trimToNull(truncateTitleToMaxFieldLength(mediaPackage.getSeriesTitle(), true));
playlistName = (playlistName == null) ? this.defaultPlaylist : playlistName;
final Playlist playlist;
final Playlist existingPlaylist = youTubeService.getMyPlaylistByTitle(playlistName);
if (existingPlaylist == null) {
playlist = youTubeService.createPlaylist(playlistName, c.getContextDescription(), playlistPrivacy,
mediaPackage.getSeries());
} else {
playlist = existingPlaylist;
}
youTubeService.addPlaylistItem(playlist.getId(), video.getId());
if (captions != null) {
for (Track caption : captions) {
try {
final File captionFile = workspace.get(caption.getURI());
String captionLanguage = null;
for (String tag : caption.getTags()) {
if (tag.startsWith("lang:")) {
captionLanguage = tag.substring(5);
break;
}
}
if (captionLanguage == null) {
logger.warn("No language tag (lang:<bcp-code>) found for caption track {}. Skipping upload.", caption);
continue;
}
// Google YouTube API requires ISO 639-1 language code.
// Opencast flavors often use that too.
youTubeService.addCaptions(video.getId(), captionFile, captionLanguage, caption.getFlavor().toString());
} catch (Exception e) {
logger.warn("Failed to upload captions {} for video {}: {}", caption, video.getId(), e.getMessage());
}
}
}
// Create new publication element
final URL url = URI.create("http://www.youtube.com/watch?v=" + video.getId() + "&list=" + playlist.getId())
.toURL();
return PublicationImpl.publication(
UUID.randomUUID().toString(), CHANNEL_NAME, url.toURI(), MimeTypes.parseMimeType(MIME_TYPE));
} catch (Exception e) {
logger.error("failed publishing to YouTube", e);
logger.warn("Error publishing {}, {}", element, e.getMessage());
if (e instanceof PublicationException) {
throw (PublicationException) e;
} else {
throw new PublicationException(
"YouTube publish failed on job: "
+ ToStringBuilder.reflectionToString(job, ToStringStyle.MULTI_LINE_STYLE),
e);
}
}
}
@Override
public Job retract(final MediaPackage mediaPackage) throws PublicationException {
if (mediaPackage == null) {
throw new IllegalArgumentException("Mediapackage must be specified");
}
try {
List<String> arguments = new ArrayList<String>();
arguments.add(MediaPackageParser.getAsXml(mediaPackage));
return serviceRegistry.createJob(JOB_TYPE, Operation.Retract.toString(), arguments, youtubeRetractJobLoad);
} catch (ServiceRegistryException e) {
throw new PublicationException("Unable to create a job", e);
}
}
/**
* Retracts the mediapackage from YouTube.
*
* @param job
* the associated job
* @param mediaPackage
* the mediapackage
* @throws PublicationException
* if retract did not work
*/
private Publication retract(final Job job, final MediaPackage mediaPackage) throws PublicationException {
logger.info("Retract video from YouTube: {}", mediaPackage);
Publication youtube = null;
for (Publication publication : mediaPackage.getPublications()) {
if (CHANNEL_NAME.equals(publication.getChannel())) {
youtube = publication;
break;
}
}
if (youtube == null) {
return null;
}
try {
final java.net.URI uri = youtube.getURI();
retract(uri.toString());
} catch (final Exception e) {
logger.error("Failure retracting YouTube media {}", e.getMessage());
throw new PublicationException("YouTube media retract failed on job: "
+ ToStringBuilder.reflectionToString(job, ToStringStyle.MULTI_LINE_STYLE), e);
}
return youtube;
}
private void retract(final String watchUrl) throws Exception {
if (watchUrl == null) {
throw new IllegalArgumentException("watchUrl must be specified");
}
String videoId = null;
String playlistId = null;
try {
final java.net.URI uri = java.net.URI.create(watchUrl);
final String query = uri.getQuery();
if (query != null) {
for (String param : query.split("&")) {
String[] pair = param.split("=", 2);
if (pair.length == 2) {
if ("v".equals(pair[0])) {
videoId = pair[1];
} else if ("list".equals(pair[0])) {
playlistId = pair[1];
}
}
}
}
} catch (Exception e) {
throw new IllegalArgumentException("Invalid YouTube watch URL: " + watchUrl, e);
}
if (videoId == null) {
throw new IllegalArgumentException("YouTube video ID not found in URL: " + watchUrl);
}
if (playlistId != null) {
try {
youTubeService.removeVideoFromPlaylist(playlistId, videoId);
} catch (Exception e) {
// Non-fatal: continue with video removal
}
}
youTubeService.removeMyVideo(videoId);
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#process(org.opencastproject.job.api.Job)
*/
@Override
protected String process(final Job job) throws Exception {
Operation op = null;
try {
op = Operation.valueOf(job.getOperation());
List<String> arguments = job.getArguments();
MediaPackage mediapackage = MediaPackageParser.getFromXml(arguments.get(0));
switch (op) {
case Publish:
String elementId = arguments.get(1);
String thumbnailId = arguments.size() > 2 ? StringUtils.trimToNull(arguments.get(2)) : null;
final List<Track> captions = new ArrayList<>();
if (arguments.size() > 3) {
for (String captionId : arguments.get(3).split(",")) {
final Track caption = mediapackage.getTrack(captionId);
if (caption != null) {
captions.add(caption);
}
}
}
Publication publicationElement = publish(job, mediapackage, elementId, thumbnailId, captions);
return (publicationElement == null) ? null : MediaPackageElementParser.getAsXml(publicationElement);
case Retract:
Publication retractedElement = retract(job, mediapackage);
return (retractedElement == null) ? null : MediaPackageElementParser.getAsXml(retractedElement);
default:
throw new IllegalStateException("Don't know how to handle operation '" + job.getOperation() + "'");
}
} catch (final IllegalArgumentException e) {
throw new ServiceRegistryException("This service can't handle operations of type '" + op + "'", e);
} catch (final IndexOutOfBoundsException e) {
throw new ServiceRegistryException("This argument list for operation '" + op + "' does not meet expectations", e);
} catch (final Exception e) {
throw new ServiceRegistryException("Error handling operation '" + op + "'", e);
}
}
/**
* Callback for the OSGi environment to set the workspace reference.
*
* @param workspace
* the workspace
*/
@Reference
protected void setWorkspace(Workspace workspace) {
this.workspace = workspace;
}
/**
* Callback for the OSGi environment to set the service registry reference.
*
* @param serviceRegistry
* the service registry
*/
@Reference
protected void setServiceRegistry(ServiceRegistry serviceRegistry) {
this.serviceRegistry = serviceRegistry;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getServiceRegistry()
*/
@Override
protected ServiceRegistry getServiceRegistry() {
return serviceRegistry;
}
/**
* Callback for setting the security service.
*
* @param securityService
* the securityService to set
*/
@Reference
public void setSecurityService(SecurityService securityService) {
this.securityService = securityService;
}
/**
* Callback for setting the user directory service.
*
* @param userDirectoryService
* the userDirectoryService to set
*/
@Reference
public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
this.userDirectoryService = userDirectoryService;
}
/**
* Sets a reference to the organization directory service.
*
* @param organizationDirectory
* the organization directory
*/
@Reference
public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectory) {
this.organizationDirectoryService = organizationDirectory;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getSecurityService()
*/
@Override
protected SecurityService getSecurityService() {
return securityService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getUserDirectoryService()
*/
@Override
protected UserDirectoryService getUserDirectoryService() {
return userDirectoryService;
}
/**
* {@inheritDoc}
*
* @see org.opencastproject.job.api.AbstractJobProducer#getOrganizationDirectoryService()
*/
@Override
protected OrganizationDirectoryService getOrganizationDirectoryService() {
return organizationDirectoryService;
}
boolean isMaxFieldLengthSet() {
return maxFieldLength != 0;
}
private String truncateTitleToMaxFieldLength(final String title, final boolean tolerateNull) {
if (StringUtils.isBlank(title) && !tolerateNull) {
throw new IllegalArgumentException("Title fields cannot be null, empty, or whitespace");
}
if (isMaxFieldLengthSet() && (title != null)) {
return StringUtils.left(title, maxFieldLength);
} else {
return title;
}
}
private void defaultMaxFieldLength(String maxFieldLength) {
if (StringUtils.isBlank(maxFieldLength)) {
this.maxFieldLength = 0;
} else {
try {
this.maxFieldLength = Integer.parseInt(maxFieldLength);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("maxFieldLength must be an integer");
}
if (this.maxFieldLength <= 0) {
throw new IllegalArgumentException("maxFieldLength must be greater than zero");
}
}
}
}