1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.workingfilerepository.impl;
23
24 import org.opencastproject.cleanup.RecursiveDirectoryCleaner;
25 import org.opencastproject.rest.RestConstants;
26 import org.opencastproject.security.api.SecurityService;
27 import org.opencastproject.serviceregistry.api.ServiceRegistry;
28 import org.opencastproject.systems.OpencastConstants;
29 import org.opencastproject.util.Checksum;
30 import org.opencastproject.util.FileSupport;
31 import org.opencastproject.util.NotFoundException;
32 import org.opencastproject.util.PathSupport;
33 import org.opencastproject.util.UrlSupport;
34 import org.opencastproject.workingfilerepository.api.PathMappable;
35 import org.opencastproject.workingfilerepository.api.WorkingFileRepository;
36
37 import org.apache.commons.codec.digest.DigestUtils;
38 import org.apache.commons.io.FileUtils;
39 import org.apache.commons.io.FilenameUtils;
40 import org.apache.commons.io.IOUtils;
41 import org.apache.commons.lang3.StringUtils;
42 import org.osgi.service.component.ComponentContext;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 import java.io.File;
47 import java.io.FileInputStream;
48 import java.io.FileOutputStream;
49 import java.io.FilenameFilter;
50 import java.io.IOException;
51 import java.io.InputStream;
52 import java.net.URI;
53 import java.net.URISyntaxException;
54 import java.nio.file.AtomicMoveNotSupportedException;
55 import java.nio.file.Files;
56 import java.nio.file.Paths;
57 import java.nio.file.StandardCopyOption;
58 import java.security.DigestInputStream;
59 import java.security.MessageDigest;
60 import java.security.NoSuchAlgorithmException;
61 import java.time.Duration;
62 import java.util.Arrays;
63 import java.util.Date;
64 import java.util.List;
65 import java.util.Map;
66 import java.util.Objects;
67 import java.util.Optional;
68
69
70
71
72
73 public class WorkingFileRepositoryImpl implements WorkingFileRepository, PathMappable {
74
75 private static final Logger logger = LoggerFactory.getLogger(WorkingFileRepositoryImpl.class);
76
77
78 public static final String MD5_EXTENSION = ".md5";
79
80
81 private static final FilenameFilter MD5_FINAME_FILTER = new FilenameFilter() {
82 public boolean accept(File dir, String name) {
83 return name.endsWith(MD5_EXTENSION);
84 }
85 };
86
87
88 public static final String WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY =
89 "org.opencastproject.working.file.repository.cleanup.period";
90
91 public static final String WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY =
92 "org.opencastproject.working.file.repository.cleanup.max.age";
93
94 private static final String WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY =
95 "org.opencastproject.working.file.repository.cleanup.collections";
96
97
98 protected ServiceRegistry remoteServiceManager;
99
100
101 protected String rootDirectory = null;
102
103
104 protected String serverUrl = null;
105
106
107 protected String servicePath = null;
108
109
110 private static final String FILENAME_REGEX_DEFAULT = "(^\\W|[^\\w-.]|\\.\\.|\\.$)";
111
112
113 private static final String FILENAME_REGEX_KEY = "filename.forbidden.pattern";
114
115
116 private String filenameRegex = FILENAME_REGEX_DEFAULT;
117
118
119
120 protected SecurityService securityService;
121
122
123 private WorkingFileRepositoryCleaner workingFileRepositoryCleaner;
124
125
126
127
128 public void activate(ComponentContext cc) throws IOException {
129 if (rootDirectory != null) {
130 return;
131 }
132
133 filenameRegex = Objects.toString(
134 cc.getProperties().get(FILENAME_REGEX_KEY),
135 FILENAME_REGEX_DEFAULT);
136 logger.debug("Configured filename forbidden pattern: {}", filenameRegex);
137
138
139 serverUrl = cc.getBundleContext().getProperty(OpencastConstants.SERVER_URL_PROPERTY);
140 if (StringUtils.isBlank(serverUrl)) {
141 throw new IllegalStateException("Server URL must be set");
142 }
143
144
145 servicePath = (String) cc.getProperties().get(RestConstants.SERVICE_PATH_PROPERTY);
146
147
148 rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty("org.opencastproject.file.repo.path"));
149 if (rootDirectory == null) {
150 String storageDir = cc.getBundleContext().getProperty("org.opencastproject.storage.dir");
151 if (storageDir == null) {
152 throw new IllegalStateException("Storage directory must be set");
153 }
154 rootDirectory = storageDir + File.separator + "files";
155 }
156
157 try {
158 createRootDirectory();
159 } catch (IOException e) {
160 logger.error("Unable to create the working file repository root directory at {}", rootDirectory);
161 throw e;
162 }
163
164
165 int garbageCollectionPeriodInSeconds = -1;
166 String period = StringUtils.trimToNull(
167 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY));
168 if (period != null) {
169 try {
170 garbageCollectionPeriodInSeconds = Integer.parseInt(period);
171 } catch (NumberFormatException e) {
172 logger.error("The garbage collection period for the working file repository is not an integer {}", period);
173 throw e;
174 }
175 }
176
177
178 int maxAgeInDays = -1;
179 String age = StringUtils.trimToNull(cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY));
180 if (age != null) {
181 try {
182 maxAgeInDays = Integer.parseInt(age);
183 } catch (NumberFormatException e) {
184 logger.error("The max age for the working file repository garbage collection is not an integer {}", age);
185 throw e;
186 }
187 }
188
189
190 List<String> collectionsToCleanUp = null;
191 String collectionsToCleanUpStr = StringUtils.trimToNull(
192 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY));
193 if (collectionsToCleanUpStr != null) {
194 collectionsToCleanUp = Arrays.asList(collectionsToCleanUpStr.split("\\s*,\\s*"));
195 }
196
197
198 if (garbageCollectionPeriodInSeconds > 0 && maxAgeInDays > 0 && collectionsToCleanUp != null) {
199 workingFileRepositoryCleaner = new WorkingFileRepositoryCleaner(this,
200 garbageCollectionPeriodInSeconds, maxAgeInDays, collectionsToCleanUp);
201 workingFileRepositoryCleaner.schedule();
202 }
203
204 logger.info(getDiskSpace());
205 }
206
207
208
209
210 public void deactivate() {
211 if (workingFileRepositoryCleaner != null) {
212 workingFileRepositoryCleaner.shutdown();
213 }
214 }
215
216
217
218
219
220
221
222
223
224
225
226 @Override
227 public String toSafeName(String fileName) {
228 var extension = FilenameUtils.getExtension(fileName)
229 .replaceAll(filenameRegex, "_");
230 var baseName = FilenameUtils.getBaseName(fileName)
231 .replaceAll(filenameRegex, "_");
232
233 if (StringUtils.isEmpty(extension)) {
234 return StringUtils.left(baseName, 255);
235 }
236 return String.format("%.255s.%.255s", baseName, extension);
237 }
238
239
240
241
242
243
244 public boolean delete(String mediaPackageID, String mediaPackageElementID) throws IOException {
245 File f;
246 try {
247 f = getFile(mediaPackageID, mediaPackageElementID);
248
249 File parentDirectory = f.getParentFile();
250 logger.debug("Attempting to delete {}", parentDirectory.getAbsolutePath());
251 FileUtils.forceDelete(parentDirectory);
252 File parentsParentDirectory = parentDirectory.getParentFile();
253 if (parentsParentDirectory.isDirectory() && parentsParentDirectory.list().length == 0) {
254 FileUtils.forceDelete(parentDirectory.getParentFile());
255 }
256 return true;
257 } catch (NotFoundException e) {
258 logger.info("Unable to delete non existing media package element {}@{}", mediaPackageElementID, mediaPackageID);
259 return false;
260 }
261 }
262
263
264
265
266
267
268 public InputStream get(String mediaPackageID, String mediaPackageElementID) throws NotFoundException, IOException {
269 File f = getFile(mediaPackageID, mediaPackageElementID);
270 logger.debug("Attempting to read file {}", f.getAbsolutePath());
271 return new FileInputStream(f);
272 }
273
274
275
276
277
278
279
280 @Override
281 public URI getCollectionURI(String collectionID, String fileName) {
282 try {
283 return new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionID + "/" + toSafeName(fileName));
284 } catch (URISyntaxException e) {
285 throw new IllegalStateException("Unable to create valid uri from " + collectionID + " and " + fileName);
286 }
287 }
288
289
290
291
292
293
294 public URI getURI(String mediaPackageID, String mediaPackageElementID) {
295 return getURI(mediaPackageID, mediaPackageElementID, null);
296 }
297
298
299
300
301
302
303
304 @Override
305 public URI getURI(String mediaPackageID, String mediaPackageElementID, String fileName) {
306 String uri = UrlSupport.concat(getBaseUri().toString(), MEDIAPACKAGE_PATH_PREFIX, mediaPackageID,
307 mediaPackageElementID);
308 if (fileName == null) {
309 File existingDirectory = getElementDirectory(mediaPackageID, mediaPackageElementID);
310 if (existingDirectory.isDirectory()) {
311 File[] files = existingDirectory.listFiles();
312 boolean md5Exists = false;
313 for (File f : files) {
314 if (f.getName().endsWith(MD5_EXTENSION)) {
315 md5Exists = true;
316 } else {
317 fileName = f.getName();
318 }
319 }
320 if (md5Exists && fileName != null) {
321 uri = UrlSupport.concat(uri, toSafeName(fileName));
322 }
323 }
324 } else {
325 uri = UrlSupport.concat(uri, toSafeName(fileName));
326 }
327 try {
328 return new URI(uri);
329 } catch (URISyntaxException e) {
330 throw new IllegalArgumentException(e);
331 }
332
333 }
334
335
336
337
338
339
340
341 public URI put(String mediaPackageID, String mediaPackageElementID, String filename, InputStream in)
342 throws IOException {
343 checkPathSafe(mediaPackageID);
344 checkPathSafe(mediaPackageElementID);
345 File dir = getElementDirectory(mediaPackageID, mediaPackageElementID);
346
347 File[] filesToDelete = null;
348
349 if (dir.exists()) {
350 filesToDelete = dir.listFiles();
351 } else {
352 logger.debug("Attempting to create a new directory at {}", dir.getAbsolutePath());
353 FileUtils.forceMkdir(dir);
354 }
355
356
357 File f = new File(dir, toSafeName(filename));
358 File md5File = getMd5File(f);
359
360
361 File fTmp = null;
362 File md5FileTmp = null;
363
364 if (f.exists()) {
365 logger.debug("Updating file {}", f.getAbsolutePath());
366 } else {
367 logger.debug("Adding file {}", f.getAbsolutePath());
368 }
369
370 FileOutputStream out = null;
371 try {
372
373 fTmp = File.createTempFile(f.getName(), ".tmp", dir);
374 md5FileTmp = File.createTempFile(md5File.getName(), ".tmp", dir);
375
376 logger.trace("Writing to new temporary file {}", fTmp.getAbsolutePath());
377
378 out = new FileOutputStream(fTmp);
379
380
381 MessageDigest messageDigest = null;
382 DigestInputStream dis = null;
383 try {
384 messageDigest = MessageDigest.getInstance("MD5");
385 dis = new DigestInputStream(in, messageDigest);
386 IOUtils.copy(dis, out);
387 } catch (NoSuchAlgorithmException e1) {
388 logger.error("Unable to create md5 message digest");
389 }
390
391
392 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
393 try {
394 FileUtils.writeStringToFile(md5FileTmp, md5);
395 } catch (IOException e) {
396 FileUtils.deleteQuietly(md5FileTmp);
397 throw e;
398 } finally {
399 IOUtils.closeQuietly(dis);
400 }
401
402 } catch (IOException e) {
403 IOUtils.closeQuietly(out);
404 FileUtils.deleteQuietly(dir);
405 throw e;
406 } finally {
407 IOUtils.closeQuietly(out);
408 IOUtils.closeQuietly(in);
409 }
410
411
412 try {
413 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.ATOMIC_MOVE);
414 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.ATOMIC_MOVE);
415 } catch (AtomicMoveNotSupportedException e) {
416 logger.trace("Atomic move not supported by this filesystem: using replace instead");
417 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.REPLACE_EXISTING);
418 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.REPLACE_EXISTING);
419 }
420
421
422 if (filesToDelete != null && filesToDelete.length > 0) {
423 for (File fileToDelete : filesToDelete) {
424 if (!fileToDelete.equals(f) && !fileToDelete.equals(md5File)
425
426
427 && !StringUtils.startsWith(fileToDelete.getName(), ".nfs")) {
428 logger.trace("delete {}", fileToDelete.getAbsolutePath());
429 if (!fileToDelete.delete() && fileToDelete.exists()) {
430 throw new IllegalStateException("Unable to delete file: " + fileToDelete.getAbsolutePath());
431 }
432 }
433 }
434 }
435
436 return getURI(mediaPackageID, mediaPackageElementID, filename);
437 }
438
439
440
441
442
443
444
445
446
447 protected File createMd5(File f) throws IOException {
448 FileInputStream md5In = null;
449 File md5File = null;
450 try {
451 md5In = new FileInputStream(f);
452 String md5 = DigestUtils.md5Hex(md5In);
453 IOUtils.closeQuietly(md5In);
454 md5File = getMd5File(f);
455 FileUtils.writeStringToFile(md5File, md5);
456 return md5File;
457 } catch (IOException e) {
458 FileUtils.deleteQuietly(md5File);
459 throw e;
460 } finally {
461 IOUtils.closeQuietly(md5In);
462 }
463 }
464
465
466
467
468
469
470
471
472
473 private File getMd5File(File f) {
474 return new File(f.getParent(), f.getName() + MD5_EXTENSION);
475 }
476
477
478
479
480
481
482
483
484 protected File getSourceFile(File md5File) {
485 return new File(md5File.getParent(), md5File.getName().substring(0, md5File.getName().length() - 4));
486 }
487
488 protected void checkPathSafe(String id) {
489 if (id == null) {
490 throw new NullPointerException("IDs can not be null");
491 }
492 if (id.indexOf("..") > -1 || id.indexOf(File.separator) > -1) {
493 throw new IllegalArgumentException("Invalid media package, element ID, or file name");
494 }
495 }
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510 protected File getFile(String mediaPackageID, String mediaPackageElementID) throws IllegalStateException,
511 NotFoundException {
512 checkPathSafe(mediaPackageID);
513 checkPathSafe(mediaPackageElementID);
514 File directory = getElementDirectory(mediaPackageID, mediaPackageElementID);
515
516 File[] md5Files = directory.listFiles(MD5_FINAME_FILTER);
517 if (md5Files == null) {
518 logger.debug("Element directory {} does not exist", directory);
519 throw new NotFoundException("Element directory " + directory + " does not exist");
520 } else if (md5Files.length == 0) {
521 logger.debug("There are no complete files in the element directory {}", directory.getAbsolutePath());
522 throw new NotFoundException("There are no complete files in the element directory "
523 + directory.getAbsolutePath());
524 } else if (md5Files.length == 1) {
525 File f = getSourceFile(md5Files[0]);
526 if (f.exists()) {
527 return f;
528 } else {
529 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
530 }
531 } else {
532 logger.error("Integrity error: Element directory {} contains more than one element", mediaPackageID + "/"
533 + mediaPackageElementID);
534 throw new IllegalStateException("Directory " + mediaPackageID + "/" + mediaPackageElementID
535 + "does not contain exactly one element");
536 }
537 }
538
539
540
541
542
543
544
545
546
547
548
549
550 public File getFileFromCollection(String collectionId, String fileName) throws NotFoundException,
551 IllegalArgumentException {
552 checkPathSafe(collectionId);
553
554 File directory = null;
555 try {
556 directory = getCollectionDirectory(collectionId, false);
557 if (directory == null) {
558
559 directory = new File(PathSupport.concat(new String[] { rootDirectory, COLLECTION_PATH_PREFIX, collectionId }));
560 throw new NotFoundException(directory.getAbsolutePath());
561 }
562 } catch (IOException e) {
563
564 }
565 File sourceFile = new File(directory, toSafeName(fileName));
566 File md5File = getMd5File(sourceFile);
567 if (!sourceFile.exists()) {
568 throw new NotFoundException(sourceFile.getAbsolutePath());
569 }
570 if (!md5File.exists()) {
571 throw new NotFoundException(md5File.getAbsolutePath());
572 }
573 return sourceFile;
574 }
575
576 private File getElementDirectory(String mediaPackageID, String mediaPackageElementID) {
577 return Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX, mediaPackageID, mediaPackageElementID).toFile();
578 }
579
580
581
582
583
584
585
586
587
588
589
590
591
592 private File getCollectionDirectory(String collectionId, boolean create) throws IOException {
593 File collectionDir = new File(
594 PathSupport.concat(new String[]{rootDirectory, COLLECTION_PATH_PREFIX, collectionId}));
595 if (!collectionDir.exists()) {
596 if (!create) {
597 return null;
598 }
599 try {
600 FileUtils.forceMkdir(collectionDir);
601 logger.debug("Created collection directory " + collectionId);
602 } catch (IOException e) {
603
604
605 if (!collectionDir.exists()) {
606 throw new IllegalStateException("Can not create collection directory" + collectionDir);
607 }
608 }
609 }
610 return collectionDir;
611 }
612
613 void createRootDirectory() throws IOException {
614 File f = new File(rootDirectory);
615 if (!f.exists()) {
616 FileUtils.forceMkdir(f);
617 }
618 }
619
620 public long getCollectionSize(String id) throws NotFoundException {
621 File collectionDir = null;
622 try {
623 collectionDir = getCollectionDirectory(id, false);
624 if (collectionDir == null || !collectionDir.canRead()) {
625 throw new NotFoundException("Can not find collection " + id);
626 }
627 } catch (IOException e) {
628
629 }
630 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
631 if (files == null) {
632 throw new IllegalArgumentException("Collection " + id + " is not a directory");
633 }
634 return files.length;
635 }
636
637 public InputStream getFromCollection(String collectionId, String fileName) throws NotFoundException, IOException {
638 File f = getFileFromCollection(collectionId, fileName);
639 if (f == null || !f.isFile()) {
640 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
641 }
642 logger.debug("Attempting to read file {}", f.getAbsolutePath());
643 return new FileInputStream(f);
644 }
645
646
647
648
649
650
651
652
653
654 @Override
655 public URI putInCollection(String collectionId, String fileName, InputStream in) throws IOException {
656 checkPathSafe(collectionId);
657 checkPathSafe(fileName);
658 File f = Paths.get(rootDirectory, COLLECTION_PATH_PREFIX, collectionId, toSafeName(fileName)).toFile();
659 logger.debug("Attempting to write a file to {}", f.getAbsolutePath());
660 FileOutputStream out = null;
661 try {
662 if (!f.exists()) {
663 logger.debug("Attempting to create a new file at {}", f.getAbsolutePath());
664 File collectionDirectory = getCollectionDirectory(collectionId, true);
665 if (!collectionDirectory.exists()) {
666 logger.debug("Attempting to create a new directory at {}", collectionDirectory.getAbsolutePath());
667 FileUtils.forceMkdir(collectionDirectory);
668 }
669 f.createNewFile();
670 } else {
671 logger.debug("Attempting to overwrite the file at {}", f.getAbsolutePath());
672 }
673 out = new FileOutputStream(f);
674
675
676 MessageDigest messageDigest = null;
677 DigestInputStream dis = null;
678 try {
679 messageDigest = MessageDigest.getInstance("MD5");
680 dis = new DigestInputStream(in, messageDigest);
681 IOUtils.copy(dis, out);
682 } catch (NoSuchAlgorithmException e1) {
683 logger.error("Unable to create md5 message digest");
684 }
685
686
687 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
688 File md5File = null;
689 try {
690 md5File = getMd5File(f);
691 FileUtils.writeStringToFile(md5File, md5);
692 } catch (IOException e) {
693 FileUtils.deleteQuietly(md5File);
694 throw e;
695 } finally {
696 IOUtils.closeQuietly(dis);
697 }
698
699 } catch (IOException e) {
700 FileUtils.deleteQuietly(f);
701 throw e;
702 } finally {
703 IOUtils.closeQuietly(out);
704 IOUtils.closeQuietly(in);
705 }
706 return getCollectionURI(collectionId, fileName);
707 }
708
709 public URI copyTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
710 String toFileName) throws NotFoundException, IOException {
711 File source = getFileFromCollection(fromCollection, fromFileName);
712 if (source == null) {
713 throw new IllegalArgumentException("Source file " + fromCollection + "/" + fromFileName + " does not exist");
714 }
715 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
716 if (!destDir.exists()) {
717
718 try {
719 FileUtils.forceMkdir(destDir);
720 } catch (IOException e) {
721 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
722 + "' : " + e);
723 }
724 }
725 File destFile;
726 try {
727 destFile = new File(destDir, toSafeName(toFileName));
728 FileSupport.link(source, destFile);
729 createMd5(destFile);
730 } catch (Exception e) {
731 FileUtils.deleteDirectory(destDir);
732 }
733 return getURI(toMediaPackage, toMediaPackageElement, toFileName);
734 }
735
736
737
738
739
740
741
742 @Override
743 public URI moveTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
744 String toFileName) throws NotFoundException, IOException {
745 File source = getFileFromCollection(fromCollection, fromFileName);
746 File sourceMd5 = getMd5File(source);
747 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
748
749 logger.debug("Moving {} from {} to {}/{}", new String[]{fromFileName, fromCollection, toMediaPackage,
750 toMediaPackageElement});
751 if (!destDir.exists()) {
752
753 try {
754 FileUtils.forceMkdir(destDir);
755 } catch (IOException e) {
756 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
757 + "' : " + e);
758 }
759 }
760
761 File dest = null;
762 try {
763 dest = getFile(toMediaPackage, toMediaPackageElement);
764 logger.debug("Removing existing file from target location at {}", dest);
765 delete(toMediaPackage, toMediaPackageElement);
766 } catch (NotFoundException e) {
767 dest = new File(getElementDirectory(toMediaPackage, toMediaPackageElement), toSafeName(toFileName));
768 }
769
770 try {
771 FileUtils.moveFile(source, dest);
772 FileUtils.moveFile(sourceMd5, getMd5File(dest));
773 } catch (IOException e) {
774 FileUtils.deleteDirectory(destDir);
775 throw new IllegalStateException("unable to copy file" + e);
776 }
777 return getURI(toMediaPackage, toMediaPackageElement, dest.getName());
778 }
779
780
781
782
783
784
785
786 @Override
787 public boolean deleteFromCollection(String collectionId, String fileName, boolean removeCollection)
788 throws IOException {
789 File f = null;
790 try {
791 f = getFileFromCollection(collectionId, fileName);
792 } catch (NotFoundException e) {
793 logger.trace("File {}/{} does not exist", collectionId, fileName);
794 return false;
795 }
796 File md5File = getMd5File(f);
797
798 if (!f.isFile()) {
799 throw new IllegalStateException(f + " is not a regular file");
800 }
801 if (!md5File.isFile()) {
802 throw new IllegalStateException(md5File + " is not a regular file");
803 }
804 if (!md5File.delete()) {
805 throw new IOException("MD5 hash " + md5File + " cannot be deleted");
806 }
807 if (!f.delete()) {
808 throw new IOException(f + " cannot be deleted");
809 }
810
811 if (removeCollection) {
812 File parentDirectory = f.getParentFile();
813 if (parentDirectory.isDirectory() && parentDirectory.list().length == 0) {
814 logger.debug("Attempting to delete empty collection directory {}", parentDirectory.getAbsolutePath());
815 try {
816 FileUtils.forceDelete(parentDirectory);
817 } catch (IOException e) {
818 logger.warn("Unable to delete empty collection directory {}", parentDirectory.getAbsolutePath());
819 return false;
820 }
821 }
822 }
823 return true;
824 }
825
826
827
828
829
830
831
832 @Override
833 public boolean deleteFromCollection(String collectionId, String fileName) throws IOException {
834 return deleteFromCollection(collectionId, fileName, false);
835 }
836
837
838
839
840
841
842 @Override
843 public URI[] getCollectionContents(String collectionId) throws NotFoundException {
844 File collectionDir = null;
845 try {
846 collectionDir = getCollectionDirectory(collectionId, false);
847 if (collectionDir == null) {
848 throw new NotFoundException(collectionId);
849 }
850 } catch (IOException e) {
851
852 }
853
854 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
855 URI[] uris = new URI[files.length];
856 for (int i = 0; i < files.length; i++) {
857 try {
858 uris[i] = new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionId + "/"
859 + toSafeName(getSourceFile(files[i]).getName()));
860 } catch (URISyntaxException e) {
861 throw new IllegalStateException("Invalid URI for " + files[i]);
862 }
863 }
864
865 return uris;
866 }
867
868
869
870
871
872
873
874 String getMediaPackageElementDigest(String mediaPackageID, String mediaPackageElementID) throws IOException,
875 IllegalStateException, NotFoundException {
876 File f = getFile(mediaPackageID, mediaPackageElementID);
877 if (f == null) {
878 throw new NotFoundException(mediaPackageID + "/" + mediaPackageElementID);
879 }
880 return getFileDigest(f);
881 }
882
883
884
885
886
887
888
889
890 private String getFileDigest(File file) throws IOException {
891 if (file == null) {
892 throw new IllegalArgumentException("File must not be null");
893 }
894 if (!file.exists() || !file.isFile()) {
895 throw new IllegalArgumentException("File " + file.getAbsolutePath() + " can not be read");
896 }
897
898
899 File md5HashFile = getMd5File(file);
900 if (file.exists()) {
901 logger.trace("Reading precalculated hash for {} from {}", file, md5HashFile.getName());
902 return FileUtils.readFileToString(md5HashFile, "utf-8");
903 }
904
905
906 InputStream in = null;
907 String md5 = null;
908 try {
909 in = new FileInputStream(file);
910 md5 = DigestUtils.md5Hex(in);
911 } finally {
912 IOUtils.closeQuietly(in);
913 }
914
915
916 try {
917 FileUtils.writeStringToFile(md5HashFile, md5, "utf-8");
918 } catch (IOException e) {
919 logger.warn("Error storing cached md5 checksum at {}", md5HashFile);
920 throw e;
921 }
922
923 return md5;
924 }
925
926
927
928
929
930
931 public Optional<Long> getTotalSpace() {
932 File f = new File(rootDirectory);
933 return Optional.of(f.getTotalSpace());
934 }
935
936
937
938
939
940
941 public Optional<Long> getUsableSpace() {
942 File f = new File(rootDirectory);
943 return Optional.of(f.getUsableSpace());
944 }
945
946
947
948
949
950
951 @Override
952 public Optional<Long> getUsedSpace() {
953 return Optional.of(FileUtils.sizeOfDirectory(new File(rootDirectory)));
954 }
955
956
957
958
959
960
961 public String getDiskSpace() {
962 int usable = Math.round(getUsableSpace().get() / 1024 / 1024 / 1024);
963 int total = Math.round(getTotalSpace().get() / 1024 / 1024 / 1024);
964 long percent = Math.round(100.0 * getUsableSpace().get() / (1 + getTotalSpace().get()));
965 return "Usable space " + usable + " Gb out of " + total + " Gb (" + percent + "%)";
966 }
967
968
969
970
971
972
973 @Override
974 public boolean cleanupOldFilesFromCollection(String collectionId, long days) throws IOException {
975 File colDir = getCollectionDirectory(collectionId, false);
976
977 if (colDir == null) {
978 logger.trace("Collection {} does not exist", collectionId);
979 return false;
980 }
981
982 logger.info("Cleaning up files older than {} days from collection {}", days, collectionId);
983
984 if (!colDir.isDirectory()) {
985 throw new IllegalStateException(colDir + " is not a directory");
986 }
987
988 long referenceTime = System.currentTimeMillis() - days * 24 * 3600 * 1000;
989 for (File f : colDir.listFiles()) {
990 long lastModified = f.lastModified();
991 logger.trace("{} last modified: {}, reference date: {}",
992 f.getName(), new Date(lastModified), new Date(referenceTime));
993 if (lastModified <= referenceTime) {
994
995 deleteFromCollection(collectionId, f.getName());
996 logger.info("Cleaned up file {} from collection {}", f.getName(), collectionId);
997 }
998 }
999
1000 return true;
1001 }
1002
1003 @Override
1004 public boolean cleanupOldFilesFromMediaPackage(long days) throws IOException {
1005 return RecursiveDirectoryCleaner.cleanDirectory(
1006 Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX),
1007 Duration.ofDays(days));
1008 }
1009
1010
1011
1012
1013
1014
1015 @Override
1016 public String getPathPrefix() {
1017 return rootDirectory;
1018 }
1019
1020
1021
1022
1023
1024
1025 @Override
1026 public String getUrlPrefix() {
1027 return getBaseUri().toString();
1028 }
1029
1030
1031
1032
1033
1034
1035 @Override
1036 public URI getBaseUri() {
1037 if (securityService.getOrganization() != null) {
1038 Map<String, String> orgProps = securityService.getOrganization().getProperties();
1039 if (orgProps != null && orgProps.containsKey(OpencastConstants.WFR_URL_ORG_PROPERTY)) {
1040 try {
1041 return new URI(UrlSupport.concat(orgProps.get(OpencastConstants.WFR_URL_ORG_PROPERTY), servicePath));
1042 } catch (URISyntaxException ex) {
1043 logger.warn("Organization working file repository URL not set, fallback to server URL");
1044 }
1045 }
1046 }
1047
1048 return URI.create(UrlSupport.concat(serverUrl, servicePath));
1049 }
1050
1051
1052
1053
1054
1055
1056 public void setRemoteServiceManager(ServiceRegistry remoteServiceManager) {
1057 this.remoteServiceManager = remoteServiceManager;
1058 }
1059
1060 public void setSecurityService(SecurityService securityService) {
1061 this.securityService = securityService;
1062 }
1063
1064 @Override
1065 public String getStorageName() {
1066 return "Working File Repository";
1067 }
1068 }