View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.storage.impl.fs;
22  
23  import static org.apache.commons.io.FilenameUtils.EXTENSION_SEPARATOR;
24  import static org.apache.commons.io.FilenameUtils.getExtension;
25  import static org.apache.commons.lang3.exception.ExceptionUtils.getMessage;
26  import static org.opencastproject.util.FileSupport.link;
27  import static org.opencastproject.util.IoSupport.file;
28  import static org.opencastproject.util.PathSupport.path;
29  import static org.opencastproject.util.data.functions.Strings.trimToNone;
30  
31  import org.opencastproject.assetmanager.api.storage.AssetStore;
32  import org.opencastproject.assetmanager.api.storage.AssetStoreException;
33  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
34  import org.opencastproject.assetmanager.api.storage.Source;
35  import org.opencastproject.assetmanager.api.storage.StoragePath;
36  import org.opencastproject.util.FileSupport;
37  import org.opencastproject.util.NotFoundException;
38  import org.opencastproject.workspace.api.Workspace;
39  
40  import org.apache.commons.io.FileUtils;
41  import org.apache.commons.io.FilenameUtils;
42  import org.slf4j.Logger;
43  import org.slf4j.LoggerFactory;
44  
45  import java.io.File;
46  import java.io.FileInputStream;
47  import java.io.FileNotFoundException;
48  import java.io.FilenameFilter;
49  import java.io.IOException;
50  import java.io.InputStream;
51  import java.net.MalformedURLException;
52  import java.net.URI;
53  import java.util.Optional;
54  
55  public abstract class AbstractFileSystemAssetStore implements AssetStore {
56    /** Log facility */
57    private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystemAssetStore.class);
58  
59    /** The store type e.g. filesystem (short-term), aws (long-term), other implementations */
60    protected String storeType = null;
61  
62    protected abstract Workspace getWorkspace();
63  
64    protected abstract String getRootDirectory();
65    protected abstract String getRootDirectory(String orgId, String mpId);
66  
67    /**
68     * Optional further handling of the complete deletion of mediapackage from the local store.
69     * This method will be called after the deletion of the mediapackage directory.
70     * @param orgId Organization ID
71     * @param mpId Mediapackage ID
72     */
73    protected abstract void onDeleteMediaPackage(String orgId, String mpId);
74  
75    @Override
76    public void put(StoragePath storagePath, Source source) throws AssetStoreException {
77      // Retrieving the file from the workspace has the advantage that in most cases the file already exists in the local
78      // working file repository. In the very few cases where the file is not in the working file repository,
79      // this strategy leads to a minor overhead because the file not only gets downloaded and stored in the file system
80      // but also a hard link needs to be created (or if that's not possible, a copy of the file.
81      final File origin = getUniqueFileFromWorkspace(source);
82      final File destination = createFile(storagePath, source);
83      try {
84        mkParent(destination);
85        link(origin, destination);
86      } catch (IOException e) {
87        logger.error("Error while linking/copying file {} to {}: {}", origin, destination, getMessage(e));
88        throw new AssetStoreException(e);
89      } finally {
90        if (origin != null) {
91          FileUtils.deleteQuietly(origin);
92        }
93      }
94    }
95  
96    private File getUniqueFileFromWorkspace(Source source) {
97      try {
98        return getWorkspace().get(source.getUri(), true);
99      } catch (NotFoundException e) {
100       logger.error("Source file '{}' does not exist", source.getUri());
101       throw new AssetStoreException(e);
102     } catch (IOException e) {
103       logger.error("Error while getting file '{}' from workspace: {}", source.getUri(), getMessage(e));
104       throw new AssetStoreException(e);
105     }
106   }
107 
108   @Override
109   public boolean copy(final StoragePath from, final StoragePath to) throws AssetStoreException {
110     var file = findStoragePathFile(from);
111     if (file.isPresent()) {
112       var f = file.get();
113 
114       final File t = createFile(to, f);
115       mkParent(t);
116       logger.debug("Copying {} to {}", f.getAbsolutePath(), t.getAbsolutePath());
117       try {
118         link(f, t, true);
119       } catch (IOException e) {
120         logger.error("Error copying archive file {} to {}", f, t);
121         throw new AssetStoreException(e);
122       }
123       return true;
124     }
125     return false;
126   }
127 
128   @Override
129   public Optional<InputStream> get(final StoragePath path) throws AssetStoreException {
130     var file = findStoragePathFile(path);
131     if (file.isPresent()) {
132       try {
133         return Optional.of(new FileInputStream(file.get()));
134       } catch (FileNotFoundException e) {
135         logger.error("Error getting archive file {}", file);
136         throw new AssetStoreException(e);
137       }
138     }
139     return Optional.empty();
140   }
141 
142   @Override
143   public boolean contains(StoragePath path) throws AssetStoreException {
144     return findStoragePathFile(path).isPresent();
145   }
146 
147   @Override
148   public boolean delete(DeletionSelector sel) throws AssetStoreException {
149     File dir = getDeletionSelectorDir(sel);
150     if (dir == null) {
151       // MediaPackage could not be found locally. This could mean
152       //   - all snapshots live in a remote asset store
153       //   - mount failed and files are temporary not available
154       //   - file was deleted out-of-band
155       //   - other fs problem
156       // In any case, we cannot continue and return "false" to indicate that the files could not be found.
157       return false;
158     }
159     try {
160       FileUtils.deleteDirectory(dir);
161       // also delete the media package directory if all versions have been deleted
162       boolean mpDirDeleted = FileSupport.deleteHierarchyIfEmpty(file(path(
163               getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId()), sel.getOrganizationId())),
164               dir.getParentFile());
165       if (mpDirDeleted) {
166         onDeleteMediaPackage(sel.getOrganizationId(), sel.getMediaPackageId());
167       }
168       return true;
169     } catch (IOException e) {
170       logger.error("Error deleting directory from archive {}", dir);
171       throw new AssetStoreException(e);
172     }
173   }
174 
175   /**
176    * Returns the directory file from a deletion selector
177    *
178    * @param sel
179    *          the deletion selector
180    * @return the directory file or null if it does not exist (e.g. MediaPackage does not exist locally)
181    */
182   private File getDeletionSelectorDir(DeletionSelector sel) {
183     final String rootPath = getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId());
184     if (rootPath == null) {
185       return null;
186     }
187     final String basePath = path(rootPath, sel.getOrganizationId(), sel.getMediaPackageId());
188     if (sel.getVersion().isPresent()) {
189       return file(basePath, sel.getVersion().get().toString());
190     }
191     return file(basePath);
192   }
193 
194   /** Create all parent directories of a file. */
195   private void mkParent(File f) {
196     mkDirs(f.getParentFile());
197   }
198 
199   /** Create this directory and all of its parents. */
200   protected void mkDirs(File d) {
201     if (d == null) {
202       return;
203     }
204     // mkdirs *may* not have succeeded if it returns false, so check if the directory exists in that case
205     if (!d.mkdirs() && !d.exists()) {
206       final String msg = "Cannot create directory " + d;
207       logger.error(msg);
208       throw new AssetStoreException(msg);
209     }
210   }
211 
212   /** Return the extension of a file. */
213   private Optional<String> extension(File f) {
214     Optional<String> opt = trimToNone(getExtension(f.getAbsolutePath()));
215     return opt.isPresent()
216         ? Optional.of(opt.get())
217         : Optional.empty();
218   }
219 
220   /** Return the extension of a URI, i.e. the extension of its path. */
221   private Optional<String> extension(URI uri) {
222     try {
223       Optional<String> opt = trimToNone(getExtension(uri.toURL().getPath()));
224       return opt.isPresent()
225           ? Optional.of(opt.get())
226           : Optional.empty();
227     } catch (MalformedURLException e) {
228       throw new Error(e);
229     }
230   }
231 
232   /** Create a file from a storage path and the extension of file <code>f</code>. */
233   private File createFile(StoragePath p, File f) {
234     return createFile(p, extension(f));
235   }
236 
237   /** Create a file from a storage path and the extension of the URI of <code>s</code>. */
238   private File createFile(StoragePath p, Source s) {
239     return createFile(p, extension(s.getUri()));
240   }
241 
242   /** Create a file from a storage path and an optional extension. */
243   private File createFile(StoragePath p, Optional<String> extension) {
244     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
245     if (rootDirectory == null) {
246       rootDirectory = getRootDirectory();
247     }
248     return file(
249             rootDirectory,
250             p.getOrganizationId(),
251             p.getMediaPackageId(),
252             p.getVersion().toString(),
253             extension.isPresent() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
254                     .getMediaPackageElementId());
255   }
256 
257   /** Returns a file from a storage path if it exists, null otherwise */
258   private File getExistingFile(StoragePath p, Optional<String> extension) {
259     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
260     if (rootDirectory == null) {
261       return null;
262     }
263     return file(
264             rootDirectory,
265             p.getOrganizationId(),
266             p.getMediaPackageId(),
267             p.getVersion().toString(),
268             extension.isPresent() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
269                     .getMediaPackageElementId());
270   }
271 
272   /**
273    * Returns a file {@link Optional} from a storage path if one is found or an empty {@link Optional}
274    *
275    * @param storagePath
276    *          the storage path
277    * @return the file {@link Optional}
278    */
279   private Optional<File> findStoragePathFile(final StoragePath storagePath) {
280     final FilenameFilter filter = new FilenameFilter() {
281       @Override
282       public boolean accept(File dir, String name) {
283         return FilenameUtils.getBaseName(name).equals(storagePath.getMediaPackageElementId());
284       }
285     };
286     final File containerDir = getExistingFile(storagePath, Optional.empty()).getParentFile();
287 
288     var files = containerDir.listFiles(filter);
289     if (files == null) {
290       return Optional.empty();
291     }
292     switch (files.length) {
293       case 0:
294         return Optional.empty();
295       case 1:
296         return Optional.of(files[0]);
297       default:
298         throw new AssetStoreException("Storage path " + files[0].getParent()
299             + "contains multiple files with the same element id!: " + storagePath.getMediaPackageElementId());
300     }
301   }
302 
303   @Override
304   public String getStoreType() {
305     return storeType;
306   }
307 
308 }