1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.workingfilerepository.impl;
23
24 import org.opencastproject.cleanup.RecursiveDirectoryCleaner;
25 import org.opencastproject.rest.RestConstants;
26 import org.opencastproject.security.api.SecurityService;
27 import org.opencastproject.serviceregistry.api.ServiceRegistry;
28 import org.opencastproject.systems.OpencastConstants;
29 import org.opencastproject.util.Checksum;
30 import org.opencastproject.util.FileSupport;
31 import org.opencastproject.util.NotFoundException;
32 import org.opencastproject.util.PathSupport;
33 import org.opencastproject.util.UrlSupport;
34 import org.opencastproject.util.jmx.JmxUtil;
35 import org.opencastproject.workingfilerepository.api.PathMappable;
36 import org.opencastproject.workingfilerepository.api.WorkingFileRepository;
37 import org.opencastproject.workingfilerepository.jmx.WorkingFileRepositoryBean;
38
39 import org.apache.commons.codec.digest.DigestUtils;
40 import org.apache.commons.io.FileUtils;
41 import org.apache.commons.io.FilenameUtils;
42 import org.apache.commons.io.IOUtils;
43 import org.apache.commons.lang3.StringUtils;
44 import org.osgi.service.component.ComponentContext;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47
48 import java.io.File;
49 import java.io.FileInputStream;
50 import java.io.FileOutputStream;
51 import java.io.FilenameFilter;
52 import java.io.IOException;
53 import java.io.InputStream;
54 import java.net.URI;
55 import java.net.URISyntaxException;
56 import java.nio.file.AtomicMoveNotSupportedException;
57 import java.nio.file.Files;
58 import java.nio.file.Paths;
59 import java.nio.file.StandardCopyOption;
60 import java.security.DigestInputStream;
61 import java.security.MessageDigest;
62 import java.security.NoSuchAlgorithmException;
63 import java.time.Duration;
64 import java.util.Arrays;
65 import java.util.Date;
66 import java.util.List;
67 import java.util.Map;
68 import java.util.Objects;
69 import java.util.Optional;
70
71 import javax.management.ObjectInstance;
72
73
74
75
76
77 public class WorkingFileRepositoryImpl implements WorkingFileRepository, PathMappable {
78
79 private static final Logger logger = LoggerFactory.getLogger(WorkingFileRepositoryImpl.class);
80
81
82 public static final String MD5_EXTENSION = ".md5";
83
84
85 private static final FilenameFilter MD5_FINAME_FILTER = new FilenameFilter() {
86 public boolean accept(File dir, String name) {
87 return name.endsWith(MD5_EXTENSION);
88 }
89 };
90
91
92 private static final String JMX_WORKING_FILE_REPOSITORY_TYPE = "WorkingFileRepository";
93
94 public static final String WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY =
95 "org.opencastproject.working.file.repository.cleanup.period";
96
97 public static final String WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY =
98 "org.opencastproject.working.file.repository.cleanup.max.age";
99
100 private static final String WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY =
101 "org.opencastproject.working.file.repository.cleanup.collections";
102
103
104 private WorkingFileRepositoryBean workingFileRepositoryBean = new WorkingFileRepositoryBean(this);
105
106
107 private ObjectInstance registeredMXBean;
108
109
110 protected ServiceRegistry remoteServiceManager;
111
112
113 protected String rootDirectory = null;
114
115
116 protected String serverUrl = null;
117
118
119 protected String servicePath = null;
120
121
122 private static final String FILENAME_REGEX_DEFAULT = "(^\\W|[^\\w-.]|\\.\\.|\\.$)";
123
124
125 private static final String FILENAME_REGEX_KEY = "filename.forbidden.pattern";
126
127
128 private String filenameRegex = FILENAME_REGEX_DEFAULT;
129
130
131
132 protected SecurityService securityService;
133
134
135 private WorkingFileRepositoryCleaner workingFileRepositoryCleaner;
136
137
138
139
140 public void activate(ComponentContext cc) throws IOException {
141 if (rootDirectory != null) {
142 return;
143 }
144
145 filenameRegex = Objects.toString(
146 cc.getProperties().get(FILENAME_REGEX_KEY),
147 FILENAME_REGEX_DEFAULT);
148 logger.debug("Configured filename forbidden pattern: {}", filenameRegex);
149
150
151 serverUrl = cc.getBundleContext().getProperty(OpencastConstants.SERVER_URL_PROPERTY);
152 if (StringUtils.isBlank(serverUrl)) {
153 throw new IllegalStateException("Server URL must be set");
154 }
155
156
157 servicePath = (String) cc.getProperties().get(RestConstants.SERVICE_PATH_PROPERTY);
158
159
160 rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty("org.opencastproject.file.repo.path"));
161 if (rootDirectory == null) {
162 String storageDir = cc.getBundleContext().getProperty("org.opencastproject.storage.dir");
163 if (storageDir == null) {
164 throw new IllegalStateException("Storage directory must be set");
165 }
166 rootDirectory = storageDir + File.separator + "files";
167 }
168
169 try {
170 createRootDirectory();
171 } catch (IOException e) {
172 logger.error("Unable to create the working file repository root directory at {}", rootDirectory);
173 throw e;
174 }
175
176 registeredMXBean = JmxUtil.registerMXBean(workingFileRepositoryBean, JMX_WORKING_FILE_REPOSITORY_TYPE);
177
178
179 int garbageCollectionPeriodInSeconds = -1;
180 String period = StringUtils.trimToNull(
181 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY));
182 if (period != null) {
183 try {
184 garbageCollectionPeriodInSeconds = Integer.parseInt(period);
185 } catch (NumberFormatException e) {
186 logger.error("The garbage collection period for the working file repository is not an integer {}", period);
187 throw e;
188 }
189 }
190
191
192 int maxAgeInDays = -1;
193 String age = StringUtils.trimToNull(cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY));
194 if (age != null) {
195 try {
196 maxAgeInDays = Integer.parseInt(age);
197 } catch (NumberFormatException e) {
198 logger.error("The max age for the working file repository garbage collection is not an integer {}", age);
199 throw e;
200 }
201 }
202
203
204 List<String> collectionsToCleanUp = null;
205 String collectionsToCleanUpStr = StringUtils.trimToNull(
206 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY));
207 if (collectionsToCleanUpStr != null) {
208 collectionsToCleanUp = Arrays.asList(collectionsToCleanUpStr.split("\\s*,\\s*"));
209 }
210
211
212 if (garbageCollectionPeriodInSeconds > 0 && maxAgeInDays > 0 && collectionsToCleanUp != null) {
213 workingFileRepositoryCleaner = new WorkingFileRepositoryCleaner(this,
214 garbageCollectionPeriodInSeconds, maxAgeInDays, collectionsToCleanUp);
215 workingFileRepositoryCleaner.schedule();
216 }
217
218 logger.info(getDiskSpace());
219 }
220
221
222
223
224 public void deactivate() {
225 JmxUtil.unregisterMXBean(registeredMXBean);
226 if (workingFileRepositoryCleaner != null) {
227 workingFileRepositoryCleaner.shutdown();
228 }
229 }
230
231
232
233
234
235
236
237
238
239
240
241 @Override
242 public String toSafeName(String fileName) {
243 var extension = FilenameUtils.getExtension(fileName)
244 .replaceAll(filenameRegex, "_");
245 var baseName = FilenameUtils.getBaseName(fileName)
246 .replaceAll(filenameRegex, "_");
247
248 if (StringUtils.isEmpty(extension)) {
249 return StringUtils.left(baseName, 255);
250 }
251 return String.format("%.255s.%.255s", baseName, extension);
252 }
253
254
255
256
257
258
259 public boolean delete(String mediaPackageID, String mediaPackageElementID) throws IOException {
260 File f;
261 try {
262 f = getFile(mediaPackageID, mediaPackageElementID);
263
264 File parentDirectory = f.getParentFile();
265 logger.debug("Attempting to delete {}", parentDirectory.getAbsolutePath());
266 FileUtils.forceDelete(parentDirectory);
267 File parentsParentDirectory = parentDirectory.getParentFile();
268 if (parentsParentDirectory.isDirectory() && parentsParentDirectory.list().length == 0) {
269 FileUtils.forceDelete(parentDirectory.getParentFile());
270 }
271 return true;
272 } catch (NotFoundException e) {
273 logger.info("Unable to delete non existing media package element {}@{}", mediaPackageElementID, mediaPackageID);
274 return false;
275 }
276 }
277
278
279
280
281
282
283 public InputStream get(String mediaPackageID, String mediaPackageElementID) throws NotFoundException, IOException {
284 File f = getFile(mediaPackageID, mediaPackageElementID);
285 logger.debug("Attempting to read file {}", f.getAbsolutePath());
286 return new FileInputStream(f);
287 }
288
289
290
291
292
293
294
295 @Override
296 public URI getCollectionURI(String collectionID, String fileName) {
297 try {
298 return new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionID + "/" + toSafeName(fileName));
299 } catch (URISyntaxException e) {
300 throw new IllegalStateException("Unable to create valid uri from " + collectionID + " and " + fileName);
301 }
302 }
303
304
305
306
307
308
309 public URI getURI(String mediaPackageID, String mediaPackageElementID) {
310 return getURI(mediaPackageID, mediaPackageElementID, null);
311 }
312
313
314
315
316
317
318
319 @Override
320 public URI getURI(String mediaPackageID, String mediaPackageElementID, String fileName) {
321 String uri = UrlSupport.concat(getBaseUri().toString(), MEDIAPACKAGE_PATH_PREFIX, mediaPackageID,
322 mediaPackageElementID);
323 if (fileName == null) {
324 File existingDirectory = getElementDirectory(mediaPackageID, mediaPackageElementID);
325 if (existingDirectory.isDirectory()) {
326 File[] files = existingDirectory.listFiles();
327 boolean md5Exists = false;
328 for (File f : files) {
329 if (f.getName().endsWith(MD5_EXTENSION)) {
330 md5Exists = true;
331 } else {
332 fileName = f.getName();
333 }
334 }
335 if (md5Exists && fileName != null) {
336 uri = UrlSupport.concat(uri, toSafeName(fileName));
337 }
338 }
339 } else {
340 uri = UrlSupport.concat(uri, toSafeName(fileName));
341 }
342 try {
343 return new URI(uri);
344 } catch (URISyntaxException e) {
345 throw new IllegalArgumentException(e);
346 }
347
348 }
349
350
351
352
353
354
355
356 public URI put(String mediaPackageID, String mediaPackageElementID, String filename, InputStream in)
357 throws IOException {
358 checkPathSafe(mediaPackageID);
359 checkPathSafe(mediaPackageElementID);
360 File dir = getElementDirectory(mediaPackageID, mediaPackageElementID);
361
362 File[] filesToDelete = null;
363
364 if (dir.exists()) {
365 filesToDelete = dir.listFiles();
366 } else {
367 logger.debug("Attempting to create a new directory at {}", dir.getAbsolutePath());
368 FileUtils.forceMkdir(dir);
369 }
370
371
372 File f = new File(dir, toSafeName(filename));
373 File md5File = getMd5File(f);
374
375
376 File fTmp = null;
377 File md5FileTmp = null;
378
379 if (f.exists()) {
380 logger.debug("Updating file {}", f.getAbsolutePath());
381 } else {
382 logger.debug("Adding file {}", f.getAbsolutePath());
383 }
384
385 FileOutputStream out = null;
386 try {
387
388 fTmp = File.createTempFile(f.getName(), ".tmp", dir);
389 md5FileTmp = File.createTempFile(md5File.getName(), ".tmp", dir);
390
391 logger.trace("Writing to new temporary file {}", fTmp.getAbsolutePath());
392
393 out = new FileOutputStream(fTmp);
394
395
396 MessageDigest messageDigest = null;
397 DigestInputStream dis = null;
398 try {
399 messageDigest = MessageDigest.getInstance("MD5");
400 dis = new DigestInputStream(in, messageDigest);
401 IOUtils.copy(dis, out);
402 } catch (NoSuchAlgorithmException e1) {
403 logger.error("Unable to create md5 message digest");
404 }
405
406
407 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
408 try {
409 FileUtils.writeStringToFile(md5FileTmp, md5);
410 } catch (IOException e) {
411 FileUtils.deleteQuietly(md5FileTmp);
412 throw e;
413 } finally {
414 IOUtils.closeQuietly(dis);
415 }
416
417 } catch (IOException e) {
418 IOUtils.closeQuietly(out);
419 FileUtils.deleteQuietly(dir);
420 throw e;
421 } finally {
422 IOUtils.closeQuietly(out);
423 IOUtils.closeQuietly(in);
424 }
425
426
427 try {
428 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.ATOMIC_MOVE);
429 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.ATOMIC_MOVE);
430 } catch (AtomicMoveNotSupportedException e) {
431 logger.trace("Atomic move not supported by this filesystem: using replace instead");
432 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.REPLACE_EXISTING);
433 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.REPLACE_EXISTING);
434 }
435
436
437 if (filesToDelete != null && filesToDelete.length > 0) {
438 for (File fileToDelete : filesToDelete) {
439 if (!fileToDelete.equals(f) && !fileToDelete.equals(md5File)
440
441
442 && !StringUtils.startsWith(fileToDelete.getName(), ".nfs")) {
443 logger.trace("delete {}", fileToDelete.getAbsolutePath());
444 if (!fileToDelete.delete() && fileToDelete.exists()) {
445 throw new IllegalStateException("Unable to delete file: " + fileToDelete.getAbsolutePath());
446 }
447 }
448 }
449 }
450
451 return getURI(mediaPackageID, mediaPackageElementID, filename);
452 }
453
454
455
456
457
458
459
460
461
462 protected File createMd5(File f) throws IOException {
463 FileInputStream md5In = null;
464 File md5File = null;
465 try {
466 md5In = new FileInputStream(f);
467 String md5 = DigestUtils.md5Hex(md5In);
468 IOUtils.closeQuietly(md5In);
469 md5File = getMd5File(f);
470 FileUtils.writeStringToFile(md5File, md5);
471 return md5File;
472 } catch (IOException e) {
473 FileUtils.deleteQuietly(md5File);
474 throw e;
475 } finally {
476 IOUtils.closeQuietly(md5In);
477 }
478 }
479
480
481
482
483
484
485
486
487
488 private File getMd5File(File f) {
489 return new File(f.getParent(), f.getName() + MD5_EXTENSION);
490 }
491
492
493
494
495
496
497
498
499 protected File getSourceFile(File md5File) {
500 return new File(md5File.getParent(), md5File.getName().substring(0, md5File.getName().length() - 4));
501 }
502
503 protected void checkPathSafe(String id) {
504 if (id == null) {
505 throw new NullPointerException("IDs can not be null");
506 }
507 if (id.indexOf("..") > -1 || id.indexOf(File.separator) > -1) {
508 throw new IllegalArgumentException("Invalid media package, element ID, or file name");
509 }
510 }
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525 protected File getFile(String mediaPackageID, String mediaPackageElementID) throws IllegalStateException,
526 NotFoundException {
527 checkPathSafe(mediaPackageID);
528 checkPathSafe(mediaPackageElementID);
529 File directory = getElementDirectory(mediaPackageID, mediaPackageElementID);
530
531 File[] md5Files = directory.listFiles(MD5_FINAME_FILTER);
532 if (md5Files == null) {
533 logger.debug("Element directory {} does not exist", directory);
534 throw new NotFoundException("Element directory " + directory + " does not exist");
535 } else if (md5Files.length == 0) {
536 logger.debug("There are no complete files in the element directory {}", directory.getAbsolutePath());
537 throw new NotFoundException("There are no complete files in the element directory "
538 + directory.getAbsolutePath());
539 } else if (md5Files.length == 1) {
540 File f = getSourceFile(md5Files[0]);
541 if (f.exists()) {
542 return f;
543 } else {
544 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
545 }
546 } else {
547 logger.error("Integrity error: Element directory {} contains more than one element", mediaPackageID + "/"
548 + mediaPackageElementID);
549 throw new IllegalStateException("Directory " + mediaPackageID + "/" + mediaPackageElementID
550 + "does not contain exactly one element");
551 }
552 }
553
554
555
556
557
558
559
560
561
562
563
564
565 public File getFileFromCollection(String collectionId, String fileName) throws NotFoundException,
566 IllegalArgumentException {
567 checkPathSafe(collectionId);
568
569 File directory = null;
570 try {
571 directory = getCollectionDirectory(collectionId, false);
572 if (directory == null) {
573
574 directory = new File(PathSupport.concat(new String[] { rootDirectory, COLLECTION_PATH_PREFIX, collectionId }));
575 throw new NotFoundException(directory.getAbsolutePath());
576 }
577 } catch (IOException e) {
578
579 }
580 File sourceFile = new File(directory, toSafeName(fileName));
581 File md5File = getMd5File(sourceFile);
582 if (!sourceFile.exists()) {
583 throw new NotFoundException(sourceFile.getAbsolutePath());
584 }
585 if (!md5File.exists()) {
586 throw new NotFoundException(md5File.getAbsolutePath());
587 }
588 return sourceFile;
589 }
590
591 private File getElementDirectory(String mediaPackageID, String mediaPackageElementID) {
592 return Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX, mediaPackageID, mediaPackageElementID).toFile();
593 }
594
595
596
597
598
599
600
601
602
603
604
605
606
607 private File getCollectionDirectory(String collectionId, boolean create) throws IOException {
608 File collectionDir = new File(
609 PathSupport.concat(new String[]{rootDirectory, COLLECTION_PATH_PREFIX, collectionId}));
610 if (!collectionDir.exists()) {
611 if (!create) {
612 return null;
613 }
614 try {
615 FileUtils.forceMkdir(collectionDir);
616 logger.debug("Created collection directory " + collectionId);
617 } catch (IOException e) {
618
619
620 if (!collectionDir.exists()) {
621 throw new IllegalStateException("Can not create collection directory" + collectionDir);
622 }
623 }
624 }
625 return collectionDir;
626 }
627
628 void createRootDirectory() throws IOException {
629 File f = new File(rootDirectory);
630 if (!f.exists()) {
631 FileUtils.forceMkdir(f);
632 }
633 }
634
635 public long getCollectionSize(String id) throws NotFoundException {
636 File collectionDir = null;
637 try {
638 collectionDir = getCollectionDirectory(id, false);
639 if (collectionDir == null || !collectionDir.canRead()) {
640 throw new NotFoundException("Can not find collection " + id);
641 }
642 } catch (IOException e) {
643
644 }
645 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
646 if (files == null) {
647 throw new IllegalArgumentException("Collection " + id + " is not a directory");
648 }
649 return files.length;
650 }
651
652 public InputStream getFromCollection(String collectionId, String fileName) throws NotFoundException, IOException {
653 File f = getFileFromCollection(collectionId, fileName);
654 if (f == null || !f.isFile()) {
655 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
656 }
657 logger.debug("Attempting to read file {}", f.getAbsolutePath());
658 return new FileInputStream(f);
659 }
660
661
662
663
664
665
666
667
668
669 @Override
670 public URI putInCollection(String collectionId, String fileName, InputStream in) throws IOException {
671 checkPathSafe(collectionId);
672 checkPathSafe(fileName);
673 File f = Paths.get(rootDirectory, COLLECTION_PATH_PREFIX, collectionId, toSafeName(fileName)).toFile();
674 logger.debug("Attempting to write a file to {}", f.getAbsolutePath());
675 FileOutputStream out = null;
676 try {
677 if (!f.exists()) {
678 logger.debug("Attempting to create a new file at {}", f.getAbsolutePath());
679 File collectionDirectory = getCollectionDirectory(collectionId, true);
680 if (!collectionDirectory.exists()) {
681 logger.debug("Attempting to create a new directory at {}", collectionDirectory.getAbsolutePath());
682 FileUtils.forceMkdir(collectionDirectory);
683 }
684 f.createNewFile();
685 } else {
686 logger.debug("Attempting to overwrite the file at {}", f.getAbsolutePath());
687 }
688 out = new FileOutputStream(f);
689
690
691 MessageDigest messageDigest = null;
692 DigestInputStream dis = null;
693 try {
694 messageDigest = MessageDigest.getInstance("MD5");
695 dis = new DigestInputStream(in, messageDigest);
696 IOUtils.copy(dis, out);
697 } catch (NoSuchAlgorithmException e1) {
698 logger.error("Unable to create md5 message digest");
699 }
700
701
702 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
703 File md5File = null;
704 try {
705 md5File = getMd5File(f);
706 FileUtils.writeStringToFile(md5File, md5);
707 } catch (IOException e) {
708 FileUtils.deleteQuietly(md5File);
709 throw e;
710 } finally {
711 IOUtils.closeQuietly(dis);
712 }
713
714 } catch (IOException e) {
715 FileUtils.deleteQuietly(f);
716 throw e;
717 } finally {
718 IOUtils.closeQuietly(out);
719 IOUtils.closeQuietly(in);
720 }
721 return getCollectionURI(collectionId, fileName);
722 }
723
724 public URI copyTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
725 String toFileName) throws NotFoundException, IOException {
726 File source = getFileFromCollection(fromCollection, fromFileName);
727 if (source == null) {
728 throw new IllegalArgumentException("Source file " + fromCollection + "/" + fromFileName + " does not exist");
729 }
730 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
731 if (!destDir.exists()) {
732
733 try {
734 FileUtils.forceMkdir(destDir);
735 } catch (IOException e) {
736 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
737 + "' : " + e);
738 }
739 }
740 File destFile;
741 try {
742 destFile = new File(destDir, toSafeName(toFileName));
743 FileSupport.link(source, destFile);
744 createMd5(destFile);
745 } catch (Exception e) {
746 FileUtils.deleteDirectory(destDir);
747 }
748 return getURI(toMediaPackage, toMediaPackageElement, toFileName);
749 }
750
751
752
753
754
755
756
757 @Override
758 public URI moveTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
759 String toFileName) throws NotFoundException, IOException {
760 File source = getFileFromCollection(fromCollection, fromFileName);
761 File sourceMd5 = getMd5File(source);
762 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
763
764 logger.debug("Moving {} from {} to {}/{}", new String[]{fromFileName, fromCollection, toMediaPackage,
765 toMediaPackageElement});
766 if (!destDir.exists()) {
767
768 try {
769 FileUtils.forceMkdir(destDir);
770 } catch (IOException e) {
771 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
772 + "' : " + e);
773 }
774 }
775
776 File dest = null;
777 try {
778 dest = getFile(toMediaPackage, toMediaPackageElement);
779 logger.debug("Removing existing file from target location at {}", dest);
780 delete(toMediaPackage, toMediaPackageElement);
781 } catch (NotFoundException e) {
782 dest = new File(getElementDirectory(toMediaPackage, toMediaPackageElement), toSafeName(toFileName));
783 }
784
785 try {
786 FileUtils.moveFile(source, dest);
787 FileUtils.moveFile(sourceMd5, getMd5File(dest));
788 } catch (IOException e) {
789 FileUtils.deleteDirectory(destDir);
790 throw new IllegalStateException("unable to copy file" + e);
791 }
792 return getURI(toMediaPackage, toMediaPackageElement, dest.getName());
793 }
794
795
796
797
798
799
800
801 @Override
802 public boolean deleteFromCollection(String collectionId, String fileName, boolean removeCollection)
803 throws IOException {
804 File f = null;
805 try {
806 f = getFileFromCollection(collectionId, fileName);
807 } catch (NotFoundException e) {
808 logger.trace("File {}/{} does not exist", collectionId, fileName);
809 return false;
810 }
811 File md5File = getMd5File(f);
812
813 if (!f.isFile()) {
814 throw new IllegalStateException(f + " is not a regular file");
815 }
816 if (!md5File.isFile()) {
817 throw new IllegalStateException(md5File + " is not a regular file");
818 }
819 if (!md5File.delete()) {
820 throw new IOException("MD5 hash " + md5File + " cannot be deleted");
821 }
822 if (!f.delete()) {
823 throw new IOException(f + " cannot be deleted");
824 }
825
826 if (removeCollection) {
827 File parentDirectory = f.getParentFile();
828 if (parentDirectory.isDirectory() && parentDirectory.list().length == 0) {
829 logger.debug("Attempting to delete empty collection directory {}", parentDirectory.getAbsolutePath());
830 try {
831 FileUtils.forceDelete(parentDirectory);
832 } catch (IOException e) {
833 logger.warn("Unable to delete empty collection directory {}", parentDirectory.getAbsolutePath());
834 return false;
835 }
836 }
837 }
838 return true;
839 }
840
841
842
843
844
845
846
847 @Override
848 public boolean deleteFromCollection(String collectionId, String fileName) throws IOException {
849 return deleteFromCollection(collectionId, fileName, false);
850 }
851
852
853
854
855
856
857 @Override
858 public URI[] getCollectionContents(String collectionId) throws NotFoundException {
859 File collectionDir = null;
860 try {
861 collectionDir = getCollectionDirectory(collectionId, false);
862 if (collectionDir == null) {
863 throw new NotFoundException(collectionId);
864 }
865 } catch (IOException e) {
866
867 }
868
869 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
870 URI[] uris = new URI[files.length];
871 for (int i = 0; i < files.length; i++) {
872 try {
873 uris[i] = new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionId + "/"
874 + toSafeName(getSourceFile(files[i]).getName()));
875 } catch (URISyntaxException e) {
876 throw new IllegalStateException("Invalid URI for " + files[i]);
877 }
878 }
879
880 return uris;
881 }
882
883
884
885
886
887
888
889 String getMediaPackageElementDigest(String mediaPackageID, String mediaPackageElementID) throws IOException,
890 IllegalStateException, NotFoundException {
891 File f = getFile(mediaPackageID, mediaPackageElementID);
892 if (f == null) {
893 throw new NotFoundException(mediaPackageID + "/" + mediaPackageElementID);
894 }
895 return getFileDigest(f);
896 }
897
898
899
900
901
902
903
904
905 private String getFileDigest(File file) throws IOException {
906 if (file == null) {
907 throw new IllegalArgumentException("File must not be null");
908 }
909 if (!file.exists() || !file.isFile()) {
910 throw new IllegalArgumentException("File " + file.getAbsolutePath() + " can not be read");
911 }
912
913
914 File md5HashFile = getMd5File(file);
915 if (file.exists()) {
916 logger.trace("Reading precalculated hash for {} from {}", file, md5HashFile.getName());
917 return FileUtils.readFileToString(md5HashFile, "utf-8");
918 }
919
920
921 InputStream in = null;
922 String md5 = null;
923 try {
924 in = new FileInputStream(file);
925 md5 = DigestUtils.md5Hex(in);
926 } finally {
927 IOUtils.closeQuietly(in);
928 }
929
930
931 try {
932 FileUtils.writeStringToFile(md5HashFile, md5, "utf-8");
933 } catch (IOException e) {
934 logger.warn("Error storing cached md5 checksum at {}", md5HashFile);
935 throw e;
936 }
937
938 return md5;
939 }
940
941
942
943
944
945
946 public Optional<Long> getTotalSpace() {
947 File f = new File(rootDirectory);
948 return Optional.of(f.getTotalSpace());
949 }
950
951
952
953
954
955
956 public Optional<Long> getUsableSpace() {
957 File f = new File(rootDirectory);
958 return Optional.of(f.getUsableSpace());
959 }
960
961
962
963
964
965
966 @Override
967 public Optional<Long> getUsedSpace() {
968 return Optional.of(FileUtils.sizeOfDirectory(new File(rootDirectory)));
969 }
970
971
972
973
974
975
976 public String getDiskSpace() {
977 int usable = Math.round(getUsableSpace().get() / 1024 / 1024 / 1024);
978 int total = Math.round(getTotalSpace().get() / 1024 / 1024 / 1024);
979 long percent = Math.round(100.0 * getUsableSpace().get() / (1 + getTotalSpace().get()));
980 return "Usable space " + usable + " Gb out of " + total + " Gb (" + percent + "%)";
981 }
982
983
984
985
986
987
988 @Override
989 public boolean cleanupOldFilesFromCollection(String collectionId, long days) throws IOException {
990 File colDir = getCollectionDirectory(collectionId, false);
991
992 if (colDir == null) {
993 logger.trace("Collection {} does not exist", collectionId);
994 return false;
995 }
996
997 logger.info("Cleaning up files older than {} days from collection {}", days, collectionId);
998
999 if (!colDir.isDirectory()) {
1000 throw new IllegalStateException(colDir + " is not a directory");
1001 }
1002
1003 long referenceTime = System.currentTimeMillis() - days * 24 * 3600 * 1000;
1004 for (File f : colDir.listFiles()) {
1005 long lastModified = f.lastModified();
1006 logger.trace("{} last modified: {}, reference date: {}",
1007 f.getName(), new Date(lastModified), new Date(referenceTime));
1008 if (lastModified <= referenceTime) {
1009
1010 deleteFromCollection(collectionId, f.getName());
1011 logger.info("Cleaned up file {} from collection {}", f.getName(), collectionId);
1012 }
1013 }
1014
1015 return true;
1016 }
1017
1018 @Override
1019 public boolean cleanupOldFilesFromMediaPackage(long days) throws IOException {
1020 return RecursiveDirectoryCleaner.cleanDirectory(
1021 Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX),
1022 Duration.ofDays(days));
1023 }
1024
1025
1026
1027
1028
1029
1030 @Override
1031 public String getPathPrefix() {
1032 return rootDirectory;
1033 }
1034
1035
1036
1037
1038
1039
1040 @Override
1041 public String getUrlPrefix() {
1042 return getBaseUri().toString();
1043 }
1044
1045
1046
1047
1048
1049
1050 @Override
1051 public URI getBaseUri() {
1052 if (securityService.getOrganization() != null) {
1053 Map<String, String> orgProps = securityService.getOrganization().getProperties();
1054 if (orgProps != null && orgProps.containsKey(OpencastConstants.WFR_URL_ORG_PROPERTY)) {
1055 try {
1056 return new URI(UrlSupport.concat(orgProps.get(OpencastConstants.WFR_URL_ORG_PROPERTY), servicePath));
1057 } catch (URISyntaxException ex) {
1058 logger.warn("Organization working file repository URL not set, fallback to server URL");
1059 }
1060 }
1061 }
1062
1063 return URI.create(UrlSupport.concat(serverUrl, servicePath));
1064 }
1065
1066
1067
1068
1069
1070
1071 public void setRemoteServiceManager(ServiceRegistry remoteServiceManager) {
1072 this.remoteServiceManager = remoteServiceManager;
1073 }
1074
1075 public void setSecurityService(SecurityService securityService) {
1076 this.securityService = securityService;
1077 }
1078
1079 }