1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.workingfilerepository.impl;
23
24 import org.opencastproject.cleanup.RecursiveDirectoryCleaner;
25 import org.opencastproject.rest.RestConstants;
26 import org.opencastproject.security.api.SecurityService;
27 import org.opencastproject.serviceregistry.api.ServiceRegistry;
28 import org.opencastproject.systems.OpencastConstants;
29 import org.opencastproject.util.Checksum;
30 import org.opencastproject.util.FileSupport;
31 import org.opencastproject.util.NotFoundException;
32 import org.opencastproject.util.PathSupport;
33 import org.opencastproject.util.UrlSupport;
34 import org.opencastproject.util.data.Option;
35 import org.opencastproject.util.jmx.JmxUtil;
36 import org.opencastproject.workingfilerepository.api.PathMappable;
37 import org.opencastproject.workingfilerepository.api.WorkingFileRepository;
38 import org.opencastproject.workingfilerepository.jmx.WorkingFileRepositoryBean;
39
40 import org.apache.commons.codec.digest.DigestUtils;
41 import org.apache.commons.io.FileUtils;
42 import org.apache.commons.io.FilenameUtils;
43 import org.apache.commons.io.IOUtils;
44 import org.apache.commons.lang3.StringUtils;
45 import org.osgi.service.component.ComponentContext;
46 import org.slf4j.Logger;
47 import org.slf4j.LoggerFactory;
48
49 import java.io.File;
50 import java.io.FileInputStream;
51 import java.io.FileOutputStream;
52 import java.io.FilenameFilter;
53 import java.io.IOException;
54 import java.io.InputStream;
55 import java.net.URI;
56 import java.net.URISyntaxException;
57 import java.nio.file.AtomicMoveNotSupportedException;
58 import java.nio.file.Files;
59 import java.nio.file.Paths;
60 import java.nio.file.StandardCopyOption;
61 import java.security.DigestInputStream;
62 import java.security.MessageDigest;
63 import java.security.NoSuchAlgorithmException;
64 import java.time.Duration;
65 import java.util.Arrays;
66 import java.util.Date;
67 import java.util.List;
68 import java.util.Map;
69 import java.util.Objects;
70
71 import javax.management.ObjectInstance;
72
73
74
75
76
77 public class WorkingFileRepositoryImpl implements WorkingFileRepository, PathMappable {
78
79 private static final Logger logger = LoggerFactory.getLogger(WorkingFileRepositoryImpl.class);
80
81
82 public static final String MD5_EXTENSION = ".md5";
83
84
85 private static final FilenameFilter MD5_FINAME_FILTER = new FilenameFilter() {
86 public boolean accept(File dir, String name) {
87 return name.endsWith(MD5_EXTENSION);
88 }
89 };
90
91
92 private static final String JMX_WORKING_FILE_REPOSITORY_TYPE = "WorkingFileRepository";
93
94 public static final String WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY = "org.opencastproject.working.file.repository.cleanup.period";
95
96 public static final String WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY = "org.opencastproject.working.file.repository.cleanup.max.age";
97
98 private static final String WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY = "org.opencastproject.working.file.repository.cleanup.collections";
99
100
101 private WorkingFileRepositoryBean workingFileRepositoryBean = new WorkingFileRepositoryBean(this);
102
103
104 private ObjectInstance registeredMXBean;
105
106
107 protected ServiceRegistry remoteServiceManager;
108
109
110 protected String rootDirectory = null;
111
112
113 protected String serverUrl = null;
114
115
116 protected String servicePath = null;
117
118
119 private static final String FILENAME_REGEX_DEFAULT = "(^\\W|[^\\w-.]|\\.\\.|\\.$)";
120
121
122 private static final String FILENAME_REGEX_KEY = "filename.forbidden.pattern";
123
124
125 private String filenameRegex = FILENAME_REGEX_DEFAULT;
126
127
128
129 protected SecurityService securityService;
130
131
132 private WorkingFileRepositoryCleaner workingFileRepositoryCleaner;
133
134
135
136
137 public void activate(ComponentContext cc) throws IOException {
138 if (rootDirectory != null)
139 return;
140
141 filenameRegex = Objects.toString(
142 cc.getProperties().get(FILENAME_REGEX_KEY),
143 FILENAME_REGEX_DEFAULT);
144 logger.debug("Configured filename forbidden pattern: {}", filenameRegex);
145
146
147 serverUrl = cc.getBundleContext().getProperty(OpencastConstants.SERVER_URL_PROPERTY);
148 if (StringUtils.isBlank(serverUrl))
149 throw new IllegalStateException("Server URL must be set");
150
151
152 servicePath = (String) cc.getProperties().get(RestConstants.SERVICE_PATH_PROPERTY);
153
154
155 rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty("org.opencastproject.file.repo.path"));
156 if (rootDirectory == null) {
157 String storageDir = cc.getBundleContext().getProperty("org.opencastproject.storage.dir");
158 if (storageDir == null) {
159 throw new IllegalStateException("Storage directory must be set");
160 }
161 rootDirectory = storageDir + File.separator + "files";
162 }
163
164 try {
165 createRootDirectory();
166 } catch (IOException e) {
167 logger.error("Unable to create the working file repository root directory at {}", rootDirectory);
168 throw e;
169 }
170
171 registeredMXBean = JmxUtil.registerMXBean(workingFileRepositoryBean, JMX_WORKING_FILE_REPOSITORY_TYPE);
172
173
174 int garbageCollectionPeriodInSeconds = -1;
175 String period = StringUtils.trimToNull(
176 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_PERIOD_KEY));
177 if (period != null) {
178 try {
179 garbageCollectionPeriodInSeconds = Integer.parseInt(period);
180 } catch (NumberFormatException e) {
181 logger.error("The garbage collection period for the working file repository is not an integer {}", period);
182 throw e;
183 }
184 }
185
186
187 int maxAgeInDays = -1;
188 String age = StringUtils.trimToNull(cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_MAX_AGE_KEY));
189 if (age != null) {
190 try {
191 maxAgeInDays = Integer.parseInt(age);
192 } catch (NumberFormatException e) {
193 logger.error("The max age for the working file repository garbage collection is not an integer {}", age);
194 throw e;
195 }
196 }
197
198
199 List<String> collectionsToCleanUp = null;
200 String collectionsToCleanUpStr = StringUtils.trimToNull(
201 cc.getBundleContext().getProperty(WORKING_FILE_REPOSITORY_CLEANUP_COLLECTIONS_KEY));
202 if (collectionsToCleanUpStr != null) {
203 collectionsToCleanUp = Arrays.asList(collectionsToCleanUpStr.split("\\s*,\\s*"));
204 }
205
206
207 if (garbageCollectionPeriodInSeconds > 0 && maxAgeInDays > 0 && collectionsToCleanUp != null) {
208 workingFileRepositoryCleaner = new WorkingFileRepositoryCleaner(this,
209 garbageCollectionPeriodInSeconds, maxAgeInDays, collectionsToCleanUp);
210 workingFileRepositoryCleaner.schedule();
211 }
212
213 logger.info(getDiskSpace());
214 }
215
216
217
218
219 public void deactivate() {
220 JmxUtil.unregisterMXBean(registeredMXBean);
221 if (workingFileRepositoryCleaner != null) {
222 workingFileRepositoryCleaner.shutdown();
223 }
224 }
225
226
227
228
229
230
231
232
233
234
235
236 @Override
237 public String toSafeName(String fileName) {
238 var extension = FilenameUtils.getExtension(fileName)
239 .replaceAll(filenameRegex, "_");
240 var baseName = FilenameUtils.getBaseName(fileName)
241 .replaceAll(filenameRegex, "_");
242
243 if (StringUtils.isEmpty(extension)) {
244 return StringUtils.left(baseName, 255);
245 }
246 return String.format("%.255s.%.255s", baseName, extension);
247 }
248
249
250
251
252
253
254 public boolean delete(String mediaPackageID, String mediaPackageElementID) throws IOException {
255 File f;
256 try {
257 f = getFile(mediaPackageID, mediaPackageElementID);
258
259 File parentDirectory = f.getParentFile();
260 logger.debug("Attempting to delete {}", parentDirectory.getAbsolutePath());
261 FileUtils.forceDelete(parentDirectory);
262 File parentsParentDirectory = parentDirectory.getParentFile();
263 if (parentsParentDirectory.isDirectory() && parentsParentDirectory.list().length == 0)
264 FileUtils.forceDelete(parentDirectory.getParentFile());
265 return true;
266 } catch (NotFoundException e) {
267 logger.info("Unable to delete non existing media package element {}@{}", mediaPackageElementID, mediaPackageID);
268 return false;
269 }
270 }
271
272
273
274
275
276
277 public InputStream get(String mediaPackageID, String mediaPackageElementID) throws NotFoundException, IOException {
278 File f = getFile(mediaPackageID, mediaPackageElementID);
279 logger.debug("Attempting to read file {}", f.getAbsolutePath());
280 return new FileInputStream(f);
281 }
282
283
284
285
286
287
288
289 @Override
290 public URI getCollectionURI(String collectionID, String fileName) {
291 try {
292 return new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionID + "/" + toSafeName(fileName));
293 } catch (URISyntaxException e) {
294 throw new IllegalStateException("Unable to create valid uri from " + collectionID + " and " + fileName);
295 }
296 }
297
298
299
300
301
302
303 public URI getURI(String mediaPackageID, String mediaPackageElementID) {
304 return getURI(mediaPackageID, mediaPackageElementID, null);
305 }
306
307
308
309
310
311
312
313 @Override
314 public URI getURI(String mediaPackageID, String mediaPackageElementID, String fileName) {
315 String uri = UrlSupport.concat(getBaseUri().toString(), MEDIAPACKAGE_PATH_PREFIX, mediaPackageID,
316 mediaPackageElementID);
317 if (fileName == null) {
318 File existingDirectory = getElementDirectory(mediaPackageID, mediaPackageElementID);
319 if (existingDirectory.isDirectory()) {
320 File[] files = existingDirectory.listFiles();
321 boolean md5Exists = false;
322 for (File f : files) {
323 if (f.getName().endsWith(MD5_EXTENSION)) {
324 md5Exists = true;
325 } else {
326 fileName = f.getName();
327 }
328 }
329 if (md5Exists && fileName != null) {
330 uri = UrlSupport.concat(uri, toSafeName(fileName));
331 }
332 }
333 } else {
334 uri = UrlSupport.concat(uri, toSafeName(fileName));
335 }
336 try {
337 return new URI(uri);
338 } catch (URISyntaxException e) {
339 throw new IllegalArgumentException(e);
340 }
341
342 }
343
344
345
346
347
348
349
350 public URI put(String mediaPackageID, String mediaPackageElementID, String filename, InputStream in)
351 throws IOException {
352 checkPathSafe(mediaPackageID);
353 checkPathSafe(mediaPackageElementID);
354 File dir = getElementDirectory(mediaPackageID, mediaPackageElementID);
355
356 File[] filesToDelete = null;
357
358 if (dir.exists()) {
359 filesToDelete = dir.listFiles();
360 } else {
361 logger.debug("Attempting to create a new directory at {}", dir.getAbsolutePath());
362 FileUtils.forceMkdir(dir);
363 }
364
365
366 File f = new File(dir, toSafeName(filename));
367 File md5File = getMd5File(f);
368
369
370 File fTmp = null;
371 File md5FileTmp = null;
372
373 if (f.exists()) {
374 logger.debug("Updating file {}", f.getAbsolutePath());
375 } else {
376 logger.debug("Adding file {}", f.getAbsolutePath());
377 }
378
379 FileOutputStream out = null;
380 try {
381
382 fTmp = File.createTempFile(f.getName(), ".tmp", dir);
383 md5FileTmp = File.createTempFile(md5File.getName(), ".tmp", dir);
384
385 logger.trace("Writing to new temporary file {}", fTmp.getAbsolutePath());
386
387 out = new FileOutputStream(fTmp);
388
389
390 MessageDigest messageDigest = null;
391 DigestInputStream dis = null;
392 try {
393 messageDigest = MessageDigest.getInstance("MD5");
394 dis = new DigestInputStream(in, messageDigest);
395 IOUtils.copy(dis, out);
396 } catch (NoSuchAlgorithmException e1) {
397 logger.error("Unable to create md5 message digest");
398 }
399
400
401 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
402 try {
403 FileUtils.writeStringToFile(md5FileTmp, md5);
404 } catch (IOException e) {
405 FileUtils.deleteQuietly(md5FileTmp);
406 throw e;
407 } finally {
408 IOUtils.closeQuietly(dis);
409 }
410
411 } catch (IOException e) {
412 IOUtils.closeQuietly(out);
413 FileUtils.deleteQuietly(dir);
414 throw e;
415 } finally {
416 IOUtils.closeQuietly(out);
417 IOUtils.closeQuietly(in);
418 }
419
420
421 try {
422 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.ATOMIC_MOVE);
423 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.ATOMIC_MOVE);
424 } catch (AtomicMoveNotSupportedException e) {
425 logger.trace("Atomic move not supported by this filesystem: using replace instead");
426 Files.move(md5FileTmp.toPath(), md5File.toPath(), StandardCopyOption.REPLACE_EXISTING);
427 Files.move(fTmp.toPath(), f.toPath(), StandardCopyOption.REPLACE_EXISTING);
428 }
429
430
431 if (filesToDelete != null && filesToDelete.length > 0) {
432 for (File fileToDelete : filesToDelete) {
433 if (!fileToDelete.equals(f) && !fileToDelete.equals(md5File)
434
435
436 && !StringUtils.startsWith(fileToDelete.getName(), ".nfs")) {
437 logger.trace("delete {}", fileToDelete.getAbsolutePath());
438 if (!fileToDelete.delete() && fileToDelete.exists()) {
439 throw new IllegalStateException("Unable to delete file: " + fileToDelete.getAbsolutePath());
440 }
441 }
442 }
443 }
444
445 return getURI(mediaPackageID, mediaPackageElementID, filename);
446 }
447
448
449
450
451
452
453
454
455
456 protected File createMd5(File f) throws IOException {
457 FileInputStream md5In = null;
458 File md5File = null;
459 try {
460 md5In = new FileInputStream(f);
461 String md5 = DigestUtils.md5Hex(md5In);
462 IOUtils.closeQuietly(md5In);
463 md5File = getMd5File(f);
464 FileUtils.writeStringToFile(md5File, md5);
465 return md5File;
466 } catch (IOException e) {
467 FileUtils.deleteQuietly(md5File);
468 throw e;
469 } finally {
470 IOUtils.closeQuietly(md5In);
471 }
472 }
473
474
475
476
477
478
479
480
481
482 private File getMd5File(File f) {
483 return new File(f.getParent(), f.getName() + MD5_EXTENSION);
484 }
485
486
487
488
489
490
491
492
493 protected File getSourceFile(File md5File) {
494 return new File(md5File.getParent(), md5File.getName().substring(0, md5File.getName().length() - 4));
495 }
496
497 protected void checkPathSafe(String id) {
498 if (id == null)
499 throw new NullPointerException("IDs can not be null");
500 if (id.indexOf("..") > -1 || id.indexOf(File.separator) > -1) {
501 throw new IllegalArgumentException("Invalid media package, element ID, or file name");
502 }
503 }
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518 protected File getFile(String mediaPackageID, String mediaPackageElementID) throws IllegalStateException,
519 NotFoundException {
520 checkPathSafe(mediaPackageID);
521 checkPathSafe(mediaPackageElementID);
522 File directory = getElementDirectory(mediaPackageID, mediaPackageElementID);
523
524 File[] md5Files = directory.listFiles(MD5_FINAME_FILTER);
525 if (md5Files == null) {
526 logger.debug("Element directory {} does not exist", directory);
527 throw new NotFoundException("Element directory " + directory + " does not exist");
528 } else if (md5Files.length == 0) {
529 logger.debug("There are no complete files in the element directory {}", directory.getAbsolutePath());
530 throw new NotFoundException("There are no complete files in the element directory " + directory.getAbsolutePath());
531 } else if (md5Files.length == 1) {
532 File f = getSourceFile(md5Files[0]);
533 if (f.exists())
534 return f;
535 else
536 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
537 } else {
538 logger.error("Integrity error: Element directory {} contains more than one element", mediaPackageID + "/"
539 + mediaPackageElementID);
540 throw new IllegalStateException("Directory " + mediaPackageID + "/" + mediaPackageElementID
541 + "does not contain exactly one element");
542 }
543 }
544
545
546
547
548
549
550
551
552
553
554
555
556 public File getFileFromCollection(String collectionId, String fileName) throws NotFoundException,
557 IllegalArgumentException {
558 checkPathSafe(collectionId);
559
560 File directory = null;
561 try {
562 directory = getCollectionDirectory(collectionId, false);
563 if (directory == null) {
564
565 directory = new File(PathSupport.concat(new String[] { rootDirectory, COLLECTION_PATH_PREFIX, collectionId }));
566 throw new NotFoundException(directory.getAbsolutePath());
567 }
568 } catch (IOException e) {
569
570 }
571 File sourceFile = new File(directory, toSafeName(fileName));
572 File md5File = getMd5File(sourceFile);
573 if (!sourceFile.exists())
574 throw new NotFoundException(sourceFile.getAbsolutePath());
575 if (!md5File.exists())
576 throw new NotFoundException(md5File.getAbsolutePath());
577 return sourceFile;
578 }
579
580 private File getElementDirectory(String mediaPackageID, String mediaPackageElementID) {
581 return Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX, mediaPackageID, mediaPackageElementID).toFile();
582 }
583
584
585
586
587
588
589
590
591
592
593
594
595
596 private File getCollectionDirectory(String collectionId, boolean create) throws IOException {
597 File collectionDir = new File(
598 PathSupport.concat(new String[]{rootDirectory, COLLECTION_PATH_PREFIX, collectionId}));
599 if (!collectionDir.exists()) {
600 if (!create)
601 return null;
602 try {
603 FileUtils.forceMkdir(collectionDir);
604 logger.debug("Created collection directory " + collectionId);
605 } catch (IOException e) {
606
607
608 if (!collectionDir.exists()) {
609 throw new IllegalStateException("Can not create collection directory" + collectionDir);
610 }
611 }
612 }
613 return collectionDir;
614 }
615
616 void createRootDirectory() throws IOException {
617 File f = new File(rootDirectory);
618 if (!f.exists())
619 FileUtils.forceMkdir(f);
620 }
621
622 public long getCollectionSize(String id) throws NotFoundException {
623 File collectionDir = null;
624 try {
625 collectionDir = getCollectionDirectory(id, false);
626 if (collectionDir == null || !collectionDir.canRead())
627 throw new NotFoundException("Can not find collection " + id);
628 } catch (IOException e) {
629
630 }
631 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
632 if (files == null)
633 throw new IllegalArgumentException("Collection " + id + " is not a directory");
634 return files.length;
635 }
636
637 public InputStream getFromCollection(String collectionId, String fileName) throws NotFoundException, IOException {
638 File f = getFileFromCollection(collectionId, fileName);
639 if (f == null || !f.isFile()) {
640 throw new NotFoundException("Unable to locate " + f + " in the working file repository");
641 }
642 logger.debug("Attempting to read file {}", f.getAbsolutePath());
643 return new FileInputStream(f);
644 }
645
646
647
648
649
650
651
652
653
654 @Override
655 public URI putInCollection(String collectionId, String fileName, InputStream in) throws IOException {
656 checkPathSafe(collectionId);
657 checkPathSafe(fileName);
658 File f = Paths.get(rootDirectory, COLLECTION_PATH_PREFIX, collectionId, toSafeName(fileName)).toFile();
659 logger.debug("Attempting to write a file to {}", f.getAbsolutePath());
660 FileOutputStream out = null;
661 try {
662 if (!f.exists()) {
663 logger.debug("Attempting to create a new file at {}", f.getAbsolutePath());
664 File collectionDirectory = getCollectionDirectory(collectionId, true);
665 if (!collectionDirectory.exists()) {
666 logger.debug("Attempting to create a new directory at {}", collectionDirectory.getAbsolutePath());
667 FileUtils.forceMkdir(collectionDirectory);
668 }
669 f.createNewFile();
670 } else {
671 logger.debug("Attempting to overwrite the file at {}", f.getAbsolutePath());
672 }
673 out = new FileOutputStream(f);
674
675
676 MessageDigest messageDigest = null;
677 DigestInputStream dis = null;
678 try {
679 messageDigest = MessageDigest.getInstance("MD5");
680 dis = new DigestInputStream(in, messageDigest);
681 IOUtils.copy(dis, out);
682 } catch (NoSuchAlgorithmException e1) {
683 logger.error("Unable to create md5 message digest");
684 }
685
686
687 String md5 = Checksum.convertToHex(dis.getMessageDigest().digest());
688 File md5File = null;
689 try {
690 md5File = getMd5File(f);
691 FileUtils.writeStringToFile(md5File, md5);
692 } catch (IOException e) {
693 FileUtils.deleteQuietly(md5File);
694 throw e;
695 } finally {
696 IOUtils.closeQuietly(dis);
697 }
698
699 } catch (IOException e) {
700 FileUtils.deleteQuietly(f);
701 throw e;
702 } finally {
703 IOUtils.closeQuietly(out);
704 IOUtils.closeQuietly(in);
705 }
706 return getCollectionURI(collectionId, fileName);
707 }
708
709 public URI copyTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
710 String toFileName) throws NotFoundException, IOException {
711 File source = getFileFromCollection(fromCollection, fromFileName);
712 if (source == null)
713 throw new IllegalArgumentException("Source file " + fromCollection + "/" + fromFileName + " does not exist");
714 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
715 if (!destDir.exists()) {
716
717 try {
718 FileUtils.forceMkdir(destDir);
719 } catch (IOException e) {
720 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
721 + "' : " + e);
722 }
723 }
724 File destFile;
725 try {
726 destFile = new File(destDir, toSafeName(toFileName));
727 FileSupport.link(source, destFile);
728 createMd5(destFile);
729 } catch (Exception e) {
730 FileUtils.deleteDirectory(destDir);
731 }
732 return getURI(toMediaPackage, toMediaPackageElement, toFileName);
733 }
734
735
736
737
738
739
740
741 @Override
742 public URI moveTo(String fromCollection, String fromFileName, String toMediaPackage, String toMediaPackageElement,
743 String toFileName) throws NotFoundException, IOException {
744 File source = getFileFromCollection(fromCollection, fromFileName);
745 File sourceMd5 = getMd5File(source);
746 File destDir = getElementDirectory(toMediaPackage, toMediaPackageElement);
747
748 logger.debug("Moving {} from {} to {}/{}", new String[]{fromFileName, fromCollection, toMediaPackage,
749 toMediaPackageElement});
750 if (!destDir.exists()) {
751
752 try {
753 FileUtils.forceMkdir(destDir);
754 } catch (IOException e) {
755 throw new IllegalStateException("could not create mediapackage/element directory '" + destDir.getAbsolutePath()
756 + "' : " + e);
757 }
758 }
759
760 File dest = null;
761 try {
762 dest = getFile(toMediaPackage, toMediaPackageElement);
763 logger.debug("Removing existing file from target location at {}", dest);
764 delete(toMediaPackage, toMediaPackageElement);
765 } catch (NotFoundException e) {
766 dest = new File(getElementDirectory(toMediaPackage, toMediaPackageElement), toSafeName(toFileName));
767 }
768
769 try {
770 FileUtils.moveFile(source, dest);
771 FileUtils.moveFile(sourceMd5, getMd5File(dest));
772 } catch (IOException e) {
773 FileUtils.deleteDirectory(destDir);
774 throw new IllegalStateException("unable to copy file" + e);
775 }
776 return getURI(toMediaPackage, toMediaPackageElement, dest.getName());
777 }
778
779
780
781
782
783
784
785 @Override
786 public boolean deleteFromCollection(String collectionId, String fileName, boolean removeCollection) throws IOException {
787 File f = null;
788 try {
789 f = getFileFromCollection(collectionId, fileName);
790 } catch (NotFoundException e) {
791 logger.trace("File {}/{} does not exist", collectionId, fileName);
792 return false;
793 }
794 File md5File = getMd5File(f);
795
796 if (!f.isFile())
797 throw new IllegalStateException(f + " is not a regular file");
798 if (!md5File.isFile())
799 throw new IllegalStateException(md5File + " is not a regular file");
800 if (!md5File.delete())
801 throw new IOException("MD5 hash " + md5File + " cannot be deleted");
802 if (!f.delete())
803 throw new IOException(f + " cannot be deleted");
804
805 if (removeCollection) {
806 File parentDirectory = f.getParentFile();
807 if (parentDirectory.isDirectory() && parentDirectory.list().length == 0) {
808 logger.debug("Attempting to delete empty collection directory {}", parentDirectory.getAbsolutePath());
809 try {
810 FileUtils.forceDelete(parentDirectory);
811 } catch (IOException e) {
812 logger.warn("Unable to delete empty collection directory {}", parentDirectory.getAbsolutePath());
813 return false;
814 }
815 }
816 }
817 return true;
818 }
819
820
821
822
823
824
825
826 @Override
827 public boolean deleteFromCollection(String collectionId, String fileName) throws IOException {
828 return deleteFromCollection(collectionId, fileName, false);
829 }
830
831
832
833
834
835
836 @Override
837 public URI[] getCollectionContents(String collectionId) throws NotFoundException {
838 File collectionDir = null;
839 try {
840 collectionDir = getCollectionDirectory(collectionId, false);
841 if (collectionDir == null)
842 throw new NotFoundException(collectionId);
843 } catch (IOException e) {
844
845 }
846
847 File[] files = collectionDir.listFiles(MD5_FINAME_FILTER);
848 URI[] uris = new URI[files.length];
849 for (int i = 0; i < files.length; i++) {
850 try {
851 uris[i] = new URI(getBaseUri() + COLLECTION_PATH_PREFIX + collectionId + "/"
852 + toSafeName(getSourceFile(files[i]).getName()));
853 } catch (URISyntaxException e) {
854 throw new IllegalStateException("Invalid URI for " + files[i]);
855 }
856 }
857
858 return uris;
859 }
860
861
862
863
864
865
866
867 String getMediaPackageElementDigest(String mediaPackageID, String mediaPackageElementID) throws IOException,
868 IllegalStateException, NotFoundException {
869 File f = getFile(mediaPackageID, mediaPackageElementID);
870 if (f == null)
871 throw new NotFoundException(mediaPackageID + "/" + mediaPackageElementID);
872 return getFileDigest(f);
873 }
874
875
876
877
878
879
880
881
882 private String getFileDigest(File file) throws IOException {
883 if (file == null)
884 throw new IllegalArgumentException("File must not be null");
885 if (!file.exists() || !file.isFile())
886 throw new IllegalArgumentException("File " + file.getAbsolutePath() + " can not be read");
887
888
889 File md5HashFile = getMd5File(file);
890 if (file.exists()) {
891 logger.trace("Reading precalculated hash for {} from {}", file, md5HashFile.getName());
892 return FileUtils.readFileToString(md5HashFile, "utf-8");
893 }
894
895
896 InputStream in = null;
897 String md5 = null;
898 try {
899 in = new FileInputStream(file);
900 md5 = DigestUtils.md5Hex(in);
901 } finally {
902 IOUtils.closeQuietly(in);
903 }
904
905
906 try {
907 FileUtils.writeStringToFile(md5HashFile, md5, "utf-8");
908 } catch (IOException e) {
909 logger.warn("Error storing cached md5 checksum at {}", md5HashFile);
910 throw e;
911 }
912
913 return md5;
914 }
915
916
917
918
919
920
921 public Option<Long> getTotalSpace() {
922 File f = new File(rootDirectory);
923 return Option.some(f.getTotalSpace());
924 }
925
926
927
928
929
930
931 public Option<Long> getUsableSpace() {
932 File f = new File(rootDirectory);
933 return Option.some(f.getUsableSpace());
934 }
935
936
937
938
939
940
941 @Override
942 public Option<Long> getUsedSpace() {
943 return Option.some(FileUtils.sizeOfDirectory(new File(rootDirectory)));
944 }
945
946
947
948
949
950
951 public String getDiskSpace() {
952 int usable = Math.round(getUsableSpace().get() / 1024 / 1024 / 1024);
953 int total = Math.round(getTotalSpace().get() / 1024 / 1024 / 1024);
954 long percent = Math.round(100.0 * getUsableSpace().get() / (1 + getTotalSpace().get()));
955 return "Usable space " + usable + " Gb out of " + total + " Gb (" + percent + "%)";
956 }
957
958
959
960
961
962
963 @Override
964 public boolean cleanupOldFilesFromCollection(String collectionId, long days) throws IOException {
965 File colDir = getCollectionDirectory(collectionId, false);
966
967 if (colDir == null) {
968 logger.trace("Collection {} does not exist", collectionId);
969 return false;
970 }
971
972 logger.info("Cleaning up files older than {} days from collection {}", days, collectionId);
973
974 if (!colDir.isDirectory())
975 throw new IllegalStateException(colDir + " is not a directory");
976
977 long referenceTime = System.currentTimeMillis() - days * 24 * 3600 * 1000;
978 for (File f : colDir.listFiles()) {
979 long lastModified = f.lastModified();
980 logger.trace("{} last modified: {}, reference date: {}",
981 f.getName(), new Date(lastModified), new Date(referenceTime));
982 if (lastModified <= referenceTime) {
983
984 deleteFromCollection(collectionId, f.getName());
985 logger.info("Cleaned up file {} from collection {}", f.getName(), collectionId);
986 }
987 }
988
989 return true;
990 }
991
992 @Override
993 public boolean cleanupOldFilesFromMediaPackage(long days) throws IOException {
994 return RecursiveDirectoryCleaner.cleanDirectory(
995 Paths.get(rootDirectory, MEDIAPACKAGE_PATH_PREFIX),
996 Duration.ofDays(days));
997 }
998
999
1000
1001
1002
1003
1004 @Override
1005 public String getPathPrefix() {
1006 return rootDirectory;
1007 }
1008
1009
1010
1011
1012
1013
1014 @Override
1015 public String getUrlPrefix() {
1016 return getBaseUri().toString();
1017 }
1018
1019
1020
1021
1022
1023
1024 @Override
1025 public URI getBaseUri() {
1026 if (securityService.getOrganization() != null) {
1027 Map<String, String> orgProps = securityService.getOrganization().getProperties();
1028 if (orgProps != null && orgProps.containsKey(OpencastConstants.WFR_URL_ORG_PROPERTY)) {
1029 try {
1030 return new URI(UrlSupport.concat(orgProps.get(OpencastConstants.WFR_URL_ORG_PROPERTY), servicePath));
1031 } catch (URISyntaxException ex) {
1032 logger.warn("Organization working file repository URL not set, fallback to server URL");
1033 }
1034 }
1035 }
1036
1037 return URI.create(UrlSupport.concat(serverUrl, servicePath));
1038 }
1039
1040
1041
1042
1043
1044
1045 public void setRemoteServiceManager(ServiceRegistry remoteServiceManager) {
1046 this.remoteServiceManager = remoteServiceManager;
1047 }
1048
1049 public void setSecurityService(SecurityService securityService) {
1050 this.securityService = securityService;
1051 }
1052
1053 }