1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.videoeditor.impl;
23
24 import static org.opencastproject.videoeditor.impl.VideoEditorProperties.SUBTITLE_GRACE_PERIOD;
25
26 import org.opencastproject.inspection.api.MediaInspectionException;
27 import org.opencastproject.inspection.api.MediaInspectionService;
28 import org.opencastproject.job.api.AbstractJobProducer;
29 import org.opencastproject.job.api.Job;
30 import org.opencastproject.job.api.JobBarrier;
31 import org.opencastproject.mediapackage.MediaPackageElementFlavor;
32 import org.opencastproject.mediapackage.MediaPackageElementParser;
33 import org.opencastproject.mediapackage.MediaPackageException;
34 import org.opencastproject.mediapackage.MediaPackageReference;
35 import org.opencastproject.mediapackage.MediaPackageReferenceImpl;
36 import org.opencastproject.mediapackage.Track;
37 import org.opencastproject.mediapackage.identifier.IdImpl;
38 import org.opencastproject.security.api.OrganizationDirectoryService;
39 import org.opencastproject.security.api.SecurityService;
40 import org.opencastproject.security.api.UserDirectoryService;
41 import org.opencastproject.serviceregistry.api.ServiceRegistry;
42 import org.opencastproject.serviceregistry.api.ServiceRegistryException;
43 import org.opencastproject.smil.api.SmilException;
44 import org.opencastproject.smil.api.SmilService;
45 import org.opencastproject.smil.entity.api.Smil;
46 import org.opencastproject.smil.entity.media.api.SmilMediaObject;
47 import org.opencastproject.smil.entity.media.container.api.SmilMediaContainer;
48 import org.opencastproject.smil.entity.media.element.api.SmilMediaElement;
49 import org.opencastproject.smil.entity.media.param.api.SmilMediaParam;
50 import org.opencastproject.smil.entity.media.param.api.SmilMediaParamGroup;
51 import org.opencastproject.subtitleparser.webvttparser.WebVTTParser;
52 import org.opencastproject.subtitleparser.webvttparser.WebVTTSubtitle;
53 import org.opencastproject.subtitleparser.webvttparser.WebVTTSubtitleCue;
54 import org.opencastproject.subtitleparser.webvttparser.WebVTTWriter;
55 import org.opencastproject.util.LoadUtil;
56 import org.opencastproject.util.NotFoundException;
57 import org.opencastproject.videoeditor.api.ProcessFailedException;
58 import org.opencastproject.videoeditor.api.VideoEditorService;
59 import org.opencastproject.videoeditor.ffmpeg.FFmpegEdit;
60 import org.opencastproject.workspace.api.Workspace;
61
62 import org.apache.commons.io.FileUtils;
63 import org.apache.commons.io.FilenameUtils;
64 import org.apache.commons.io.IOUtils;
65 import org.osgi.service.cm.ConfigurationException;
66 import org.osgi.service.cm.ManagedService;
67 import org.osgi.service.component.ComponentContext;
68 import org.osgi.service.component.annotations.Activate;
69 import org.osgi.service.component.annotations.Component;
70 import org.osgi.service.component.annotations.Deactivate;
71 import org.osgi.service.component.annotations.Reference;
72 import org.slf4j.Logger;
73 import org.slf4j.LoggerFactory;
74
75 import java.io.File;
76 import java.io.FileInputStream;
77 import java.io.FileOutputStream;
78 import java.io.IOException;
79 import java.io.InputStream;
80 import java.net.URI;
81 import java.net.URISyntaxException;
82 import java.util.ArrayList;
83 import java.util.Arrays;
84 import java.util.Dictionary;
85 import java.util.Enumeration;
86 import java.util.Iterator;
87 import java.util.LinkedList;
88 import java.util.List;
89 import java.util.Properties;
90
91 import javax.xml.bind.JAXBException;
92
93
94
95
96 @Component(
97 immediate = true,
98 service = { VideoEditorService.class,ManagedService.class },
99 property = {
100 "service.description=Video Editor Service"
101 }
102 )
103 public class VideoEditorServiceImpl extends AbstractJobProducer implements VideoEditorService, ManagedService {
104
105 public static final String JOB_LOAD_KEY = "job.load.videoeditor";
106
107 private static final float DEFAULT_JOB_LOAD = 0.8f;
108
109 private float jobload = DEFAULT_JOB_LOAD;
110
111 public static final String SEGMENTS_MIN_DURATION_KEY = "segments.min.duration";
112
113 private static final int DEFAULT_SEGMENTS_MIN_DURATION = 2000;
114
115 private int segmentsMinDuration = DEFAULT_SEGMENTS_MIN_DURATION;
116
117 public static final String SEGMENTS_MIN_CUT_DURATION_KEY = "segments.min.cut.duration";
118
119 private static final int DEFAULT_SEGMENTS_MIN_CUT_DURATION = 2000;
120
121 private int segmentsMinCutDuration = DEFAULT_SEGMENTS_MIN_CUT_DURATION;
122
123
124
125
126 private static final Logger logger = LoggerFactory.getLogger(VideoEditorServiceImpl.class);
127 private static final String JOB_TYPE = "org.opencastproject.videoeditor";
128 private static final String COLLECTION_ID = "videoeditor";
129 private static final String SINK_FLAVOR_SUBTYPE = "trimmed";
130
131 private enum Operation {
132 PROCESS_SMIL
133 }
134
135
136
137
138 private MediaInspectionService inspectionService = null;
139
140
141
142 private Workspace workspace = null;
143
144
145
146 private ServiceRegistry serviceRegistry;
147
148
149
150 protected OrganizationDirectoryService organizationDirectoryService = null;
151
152
153
154 protected SecurityService securityService = null;
155
156
157
158 protected UserDirectoryService userDirectoryService = null;
159
160
161
162 protected SmilService smilService = null;
163
164
165
166 private Properties properties = new Properties();
167
168 public VideoEditorServiceImpl() {
169 super(JOB_TYPE);
170 }
171
172
173
174
175
176
177
178
179
180
181
182
183
184 protected Track processSmil(Job job, Smil smil, String trackParamGroupId) throws ProcessFailedException {
185
186 SmilMediaParamGroup trackParamGroup;
187 ArrayList<String> inputfile = new ArrayList<>();
188 ArrayList<VideoClip> videoclips = new ArrayList<>();
189 ArrayList<VideoClip> refElements = new ArrayList<>();
190 try {
191 trackParamGroup = (SmilMediaParamGroup) smil.get(trackParamGroupId);
192 } catch (SmilException ex) {
193
194 throw new ProcessFailedException("Smil does not contain a paramGroup element with Id " + trackParamGroupId);
195 }
196 MediaPackageElementFlavor sourceTrackFlavor = null;
197 String sourceTrackUri = null;
198 MediaPackageReference ref = null;
199
200 for (SmilMediaParam param : trackParamGroup.getParams()) {
201 if (SmilMediaParam.PARAM_NAME_TRACK_SRC.equals(param.getName())) {
202 sourceTrackUri = param.getValue();
203 } else if (SmilMediaParam.PARAM_NAME_TRACK_FLAVOR.equals(param.getName())) {
204 sourceTrackFlavor = MediaPackageElementFlavor.parseFlavor(param.getValue());
205 } else if (SmilMediaParam.PARAM_NAME_TRACK_ID.equals(param.getName())) {
206 ref = new MediaPackageReferenceImpl("track", param.getValue());
207 }
208 }
209 File sourceFile;
210 try {
211 sourceFile = workspace.get(new URI(sourceTrackUri));
212 } catch (IOException ex) {
213 throw new ProcessFailedException("Can't read " + sourceTrackUri);
214 } catch (NotFoundException ex) {
215 throw new ProcessFailedException("Workspace does not contain a track " + sourceTrackUri);
216 } catch (URISyntaxException ex) {
217 throw new ProcessFailedException("Source URI " + sourceTrackUri + " is not valid.");
218 }
219
220 Job inspectionJob;
221 Track sourceTrack;
222 try {
223 inspectionJob = inspect(job, new URI(sourceTrackUri));
224 sourceTrack = (Track) MediaPackageElementParser.getFromXml(inspectionJob.getPayload());
225 } catch (URISyntaxException e) {
226 throw new ProcessFailedException("Source URI " + sourceTrackUri + " is not valid.");
227 } catch (MediaInspectionException e) {
228 throw new ProcessFailedException("Media inspection of " + sourceTrackUri + " failed", e);
229 } catch (MediaPackageException e) {
230 throw new ProcessFailedException("Deserialization of source track " + sourceTrackUri + " failed", e);
231 }
232
233
234 File tempDirectory = new File(new File(workspace.rootDirectory()), "editor");
235 tempDirectory = new File(tempDirectory, Long.toString(job.getId()));
236
237 URI newTrackURI;
238 inputfile.add(sourceFile.getAbsolutePath());
239 int srcIndex = inputfile.indexOf(sourceFile.getAbsolutePath());
240 logger.info("Start processing srcfile {}", sourceFile.getAbsolutePath());
241 try {
242
243 for (SmilMediaObject element : smil.getBody().getMediaElements()) {
244
245 if (element.isContainer()) {
246 SmilMediaContainer container = (SmilMediaContainer) element;
247 if (SmilMediaContainer.ContainerType.PAR == container.getContainerType()) {
248
249 for (SmilMediaObject elementChild : container.getElements()) {
250 if (!elementChild.isContainer()) {
251 SmilMediaElement media = (SmilMediaElement) elementChild;
252 if (trackParamGroupId.equals(media.getParamGroup())) {
253 long begin = media.getClipBeginMS();
254 long end = media.getClipEndMS();
255 URI clipTrackURI = media.getSrc();
256 File clipSourceFile = null;
257 if (clipTrackURI != null) {
258 try {
259 clipSourceFile = workspace.get(clipTrackURI);
260 } catch (IOException ex) {
261 throw new ProcessFailedException("Can't read " + clipTrackURI);
262 } catch (NotFoundException ex) {
263 throw new ProcessFailedException("Workspace does not contain a track " + clipTrackURI);
264 }
265 }
266 int index;
267
268 if (clipSourceFile != null) {
269 index = inputfile.indexOf(clipSourceFile.getAbsolutePath());
270 if (index == -1) {
271 inputfile.add(clipSourceFile.getAbsolutePath());
272
273 }
274 index = inputfile.indexOf(clipSourceFile.getAbsolutePath());
275 } else {
276 index = srcIndex;
277 }
278
279
280 if (media.getMediaType() == SmilMediaElement.MediaType.REF) {
281 refElements.add(new VideoClip(index, begin, end));
282 } else {
283 videoclips.add(new VideoClip(index, begin, end));
284 }
285 }
286 } else {
287 throw new ProcessFailedException("Smil container '"
288 + ((SmilMediaContainer) elementChild).getContainerType().toString()
289 + "'is not supported yet");
290 }
291 }
292 } else {
293 throw new ProcessFailedException("Smil container '"
294 + container.getContainerType().toString() + "'is not supported yet");
295 }
296 }
297 }
298
299
300 if (videoclips.size() > 0 && refElements.size() > 0) {
301 throw new ProcessFailedException("Can not process media elements together with ref elements. "
302 + "There likely is an error in the SMIL file");
303 }
304
305
306 String outputFileExtension = null;
307 if (videoclips.size() > 0) {
308 outputFileExtension = properties.getProperty(VideoEditorProperties.DEFAULT_EXTENSION, ".mp4");
309 }
310 if (refElements.size() > 0) {
311 String extension = FilenameUtils.getExtension(sourceTrackUri);
312 if (VideoEditorProperties.WEBVTT_EXTENSION.equals(extension)) {
313 outputFileExtension = properties.getProperty(VideoEditorProperties.WEBVTT_EXTENSION, ".vtt");
314 }
315 }
316 outputFileExtension = properties.getProperty(VideoEditorProperties.OUTPUT_FILE_EXTENSION, outputFileExtension);
317
318 if (!outputFileExtension.startsWith(".")) {
319 outputFileExtension = '.' + outputFileExtension;
320 }
321
322 String filename = String.format("%s-%s%s", sourceTrackFlavor,
323 FilenameUtils.removeExtension(sourceFile.getName()), outputFileExtension);
324 File outputPath = new File(tempDirectory, filename);
325
326 if (!outputPath.getParentFile().exists()) {
327 outputPath.getParentFile().mkdirs();
328 }
329
330
331 if (videoclips.size() > 0) {
332
333 List<VideoClip> cleanclips = sortSegments(videoclips, segmentsMinDuration, segmentsMinCutDuration);
334 String error = null;
335 String outputResolution = "";
336
337
338
339 FFmpegEdit ffmpeg = new FFmpegEdit(properties);
340 error = ffmpeg.processEdits(inputfile, outputPath.getAbsolutePath(), outputResolution, cleanclips,
341 sourceTrack.hasAudio(), sourceTrack.hasVideo());
342
343 if (error != null) {
344 FileUtils.deleteQuietly(tempDirectory);
345 throw new ProcessFailedException("Editing pipeline exited abnormally! Error: " + error);
346 }
347 }
348
349
350
351
352 if (refElements.size() > 0) {
353
354 List<VideoClip> cleanclips = sortSegments(refElements, segmentsMinDuration, segmentsMinCutDuration);
355 String extension = FilenameUtils.getExtension(sourceTrackUri);
356 if (VideoEditorProperties.WEBVTT_EXTENSION.equals(extension)) {
357
358 WebVTTParser parser = new WebVTTParser();
359 WebVTTSubtitle subtitle;
360 try (FileInputStream fin = new FileInputStream(sourceFile)) {
361 subtitle = parser.parse(fin);
362 }
363
364
365 List<WebVTTSubtitleCue> cutCues = new ArrayList<>();
366 double removedTime = 0;
367 for (int i = 0; i < cleanclips.size(); i++) {
368 if (i == 0) {
369 removedTime = removedTime
370 + cleanclips.get(i).getStartInMilliseconds();
371 } else {
372 removedTime = removedTime
373 + cleanclips.get(i).getStartInMilliseconds()
374 - cleanclips.get(i - 1).getEndInMilliseconds();
375 }
376 for (WebVTTSubtitleCue cue : subtitle.getCues()) {
377 if ((cleanclips.get(i).getStartInMilliseconds() - SUBTITLE_GRACE_PERIOD) <= cue.getStartTime()
378 && (cleanclips.get(i).getEndInMilliseconds() + SUBTITLE_GRACE_PERIOD) >= cue.getEndTime()) {
379 cue.setStartTime((long) (cue.getStartTime() - removedTime));
380 cue.setEndTime((long) (cue.getEndTime() - removedTime));
381 cutCues.add(cue);
382 }
383 }
384 }
385 subtitle.setCues(cutCues);
386
387
388 try (FileOutputStream fos = new FileOutputStream(outputPath)) {
389 WebVTTWriter writer = new WebVTTWriter();
390 writer.write(subtitle, fos);
391 }
392 } else {
393 throw new ProcessFailedException("The video editor does not support the following file: " + sourceTrackUri);
394 }
395 }
396
397
398 String newTrackId = IdImpl.fromUUID().toString();
399 InputStream in = new FileInputStream(outputPath);
400 try {
401 newTrackURI = workspace.putInCollection(COLLECTION_ID,
402 String.format("%s-%s%s", sourceTrackFlavor.getType(), newTrackId, outputFileExtension), in);
403 } catch (IllegalArgumentException ex) {
404 throw new ProcessFailedException("Copy track into workspace failed! " + ex.getMessage());
405 } finally {
406 IOUtils.closeQuietly(in);
407 FileUtils.deleteQuietly(tempDirectory);
408 }
409
410
411 try {
412 inspectionJob = inspect(job,newTrackURI);
413 } catch (MediaInspectionException e) {
414 throw new ProcessFailedException("Media inspection of " + newTrackURI + " failed", e);
415 }
416 Track editedTrack = (Track) MediaPackageElementParser.getFromXml(inspectionJob.getPayload());
417 logger.info("Finished editing track {}", editedTrack);
418 editedTrack.setIdentifier(newTrackId);
419 editedTrack.setReference(ref);
420 if (videoclips.size() > 0) {
421 editedTrack.setFlavor(new MediaPackageElementFlavor(sourceTrackFlavor.getType(), SINK_FLAVOR_SUBTYPE));
422 }
423 if (refElements.size() > 0) {
424 String extension = FilenameUtils.getExtension(sourceTrackUri);
425 if (VideoEditorProperties.WEBVTT_EXTENSION.equals(extension)) {
426 editedTrack.setFlavor(new MediaPackageElementFlavor(sourceTrackFlavor.getType(),
427 sourceTrackFlavor.getSubtype() + "+" + SINK_FLAVOR_SUBTYPE));
428 }
429 }
430
431 return editedTrack;
432
433 } catch (MediaInspectionException ex) {
434 throw new ProcessFailedException("Inspecting encoded Track failed with: " + ex.getMessage());
435 } catch (MediaPackageException ex) {
436 throw new ProcessFailedException("Unable to serialize edited Track! " + ex.getMessage());
437 } catch (Exception ex) {
438 throw new ProcessFailedException("Unable to process SMIL: " + ex.getMessage(), ex);
439 } finally {
440 FileUtils.deleteQuietly(tempDirectory);
441 }
442 }
443
444
445
446
447 protected Job inspect(Job job, URI workspaceURI) throws MediaInspectionException, ProcessFailedException {
448 Job inspectionJob;
449 try {
450 inspectionJob = inspectionService.inspect(workspaceURI);
451 } catch (MediaInspectionException e) {
452 incident().recordJobCreationIncident(job, e);
453 throw new MediaInspectionException("Media inspection of " + workspaceURI + " failed", e);
454 }
455 JobBarrier barrier = new JobBarrier(job, serviceRegistry, inspectionJob);
456 if (!barrier.waitForJobs().isSuccess()) {
457 throw new ProcessFailedException("Media inspection of " + workspaceURI + " failed");
458 }
459 return inspectionJob;
460 }
461
462
463
464
465 private static List<VideoClip> sortSegments(List<VideoClip> edits, int segmentsMinDuration,
466 int segmentsMinCutDuration) {
467 LinkedList<VideoClip> ll = new LinkedList<>();
468 List<VideoClip> clips = new ArrayList<>();
469 Iterator<VideoClip> it = edits.iterator();
470 VideoClip clip;
471 VideoClip nextclip;
472 while (it.hasNext()) {
473 clip = it.next();
474 if (clip.getDurationInMilliseconds() > segmentsMinDuration) {
475 ll.add(clip);
476 }
477 }
478 clip = ll.pop();
479 while (!ll.isEmpty()) {
480 if (ll.peek() != null) {
481 nextclip = ll.pop();
482
483 if (nextclip.getSrc() == clip.getSrc()
484 && nextclip.getStartInMilliseconds() - clip.getEndInMilliseconds() < segmentsMinCutDuration) {
485 clip.setEnd(nextclip.getEndInMilliseconds());
486 } else {
487 clips.add(clip);
488 clip = nextclip;
489 }
490 }
491 }
492 clips.add(clip);
493 return clips;
494 }
495
496
497
498
499
500
501
502 @Override
503 public List<Job> processSmil(Smil smil) throws ProcessFailedException {
504 if (smil == null) {
505 throw new ProcessFailedException("Smil document is null!");
506 }
507
508 List<Job> jobs = new LinkedList<Job>();
509 try {
510 for (SmilMediaParamGroup paramGroup : smil.getHead().getParamGroups()) {
511 for (SmilMediaParam param : paramGroup.getParams()) {
512 if (SmilMediaParam.PARAM_NAME_TRACK_ID.equals(param.getName())) {
513 jobs.add(serviceRegistry.createJob(getJobType(), Operation.PROCESS_SMIL.toString(),
514 Arrays.asList(smil.toXML(), paramGroup.getId()), jobload));
515 }
516 }
517 }
518 return jobs;
519 } catch (JAXBException ex) {
520 throw new ProcessFailedException("Failed to serialize smil " + smil.getId());
521 } catch (ServiceRegistryException ex) {
522 throw new ProcessFailedException("Failed to create job: " + ex.getMessage());
523 } catch (Exception ex) {
524 throw new ProcessFailedException(ex.getMessage());
525 }
526 }
527
528 @Override
529 protected String process(Job job) throws Exception {
530 if (Operation.PROCESS_SMIL.toString().equals(job.getOperation())) {
531 Smil smil = smilService.fromXml(job.getArguments().get(0)).getSmil();
532 if (smil == null) {
533 throw new ProcessFailedException("Smil document is null!");
534 }
535
536 Track editedTrack = processSmil(job, smil, job.getArguments().get(1));
537 return MediaPackageElementParser.getAsXml(editedTrack);
538 }
539
540 throw new ProcessFailedException("Can't handle this operation: " + job.getOperation());
541 }
542
543 @Override
544 protected ServiceRegistry getServiceRegistry() {
545 return serviceRegistry;
546 }
547
548 @Override
549 protected SecurityService getSecurityService() {
550 return securityService;
551 }
552
553 @Override
554 protected UserDirectoryService getUserDirectoryService() {
555 return userDirectoryService;
556 }
557
558 @Override
559 protected OrganizationDirectoryService getOrganizationDirectoryService() {
560 return organizationDirectoryService;
561 }
562
563 @Override
564 @Activate
565 public void activate(ComponentContext context) {
566 logger.debug("activating...");
567 super.activate(context);
568 FFmpegEdit.init(context.getBundleContext());
569 }
570
571 @Deactivate
572 protected void deactivate(ComponentContext context) {
573 logger.debug("deactivating...");
574 }
575
576 @Override
577 public void updated(Dictionary<String, ?> properties) throws ConfigurationException {
578 this.properties = new Properties();
579 if (properties == null) {
580 logger.info("No configuration available, using defaults");
581 return;
582 }
583
584 Enumeration<String> keys = properties.keys();
585 while (keys.hasMoreElements()) {
586 String key = keys.nextElement();
587 this.properties.put(key, properties.get(key));
588 }
589 logger.debug("Properties updated!");
590
591 jobload = LoadUtil.getConfiguredLoadValue(properties, JOB_LOAD_KEY, DEFAULT_JOB_LOAD, serviceRegistry);
592 segmentsMinDuration = Integer.parseInt(this.properties.getProperty(SEGMENTS_MIN_DURATION_KEY,
593 String.valueOf(DEFAULT_SEGMENTS_MIN_DURATION)));
594 segmentsMinCutDuration = Integer.parseInt(this.properties.getProperty(SEGMENTS_MIN_CUT_DURATION_KEY,
595 String.valueOf(DEFAULT_SEGMENTS_MIN_CUT_DURATION)));
596 }
597
598 @Reference
599 public void setMediaInspectionService(MediaInspectionService inspectionService) {
600 this.inspectionService = inspectionService;
601 }
602
603 @Reference
604 public void setServiceRegistry(ServiceRegistry serviceRegistry) {
605 this.serviceRegistry = serviceRegistry;
606 }
607
608 @Reference
609 public void setWorkspace(Workspace workspace) {
610 this.workspace = workspace;
611 }
612
613 @Reference
614 public void setSecurityService(SecurityService securityService) {
615 this.securityService = securityService;
616 }
617
618 @Reference
619 public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
620 this.userDirectoryService = userDirectoryService;
621 }
622
623 @Reference
624 public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectoryService) {
625 this.organizationDirectoryService = organizationDirectoryService;
626 }
627
628 @Reference
629 public void setSmilService(SmilService smilService) {
630 this.smilService = smilService;
631 }
632 }