View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.storage.impl.fs;
22  
23  import static org.apache.commons.io.FilenameUtils.EXTENSION_SEPARATOR;
24  import static org.apache.commons.io.FilenameUtils.getExtension;
25  import static org.apache.commons.lang3.exception.ExceptionUtils.getMessage;
26  import static org.opencastproject.util.FileSupport.link;
27  import static org.opencastproject.util.IoSupport.file;
28  import static org.opencastproject.util.PathSupport.path;
29  import static org.opencastproject.util.data.functions.Strings.trimToNone;
30  
31  import org.opencastproject.assetmanager.api.storage.AssetStore;
32  import org.opencastproject.assetmanager.api.storage.AssetStoreException;
33  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
34  import org.opencastproject.assetmanager.api.storage.Source;
35  import org.opencastproject.assetmanager.api.storage.StoragePath;
36  import org.opencastproject.util.FileSupport;
37  import org.opencastproject.util.NotFoundException;
38  import org.opencastproject.util.data.Option;
39  import org.opencastproject.workspace.api.Workspace;
40  
41  import com.entwinemedia.fn.data.Opt;
42  
43  import org.apache.commons.io.FileUtils;
44  import org.apache.commons.io.FilenameUtils;
45  import org.slf4j.Logger;
46  import org.slf4j.LoggerFactory;
47  
48  import java.io.File;
49  import java.io.FileInputStream;
50  import java.io.FileNotFoundException;
51  import java.io.FilenameFilter;
52  import java.io.IOException;
53  import java.io.InputStream;
54  import java.net.MalformedURLException;
55  import java.net.URI;
56  import java.util.Optional;
57  
58  public abstract class AbstractFileSystemAssetStore implements AssetStore {
59    /** Log facility */
60    private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystemAssetStore.class);
61  
62    /** The store type e.g. filesystem (short-term), aws (long-term), other implementations */
63    protected String storeType = null;
64  
65    protected abstract Workspace getWorkspace();
66  
67    protected abstract String getRootDirectory();
68    protected abstract String getRootDirectory(String orgId, String mpId);
69  
70    /**
71     * Optional further handling of the complete deletion of mediapackage from the local store.
72     * This method will be called after the deletion of the mediapackage directory.
73     * @param orgId Organization ID
74     * @param mpId Mediapackage ID
75     */
76    protected abstract void onDeleteMediaPackage(String orgId, String mpId);
77  
78    @Override
79    public void put(StoragePath storagePath, Source source) throws AssetStoreException {
80      // Retrieving the file from the workspace has the advantage that in most cases the file already exists in the local
81      // working file repository. In the very few cases where the file is not in the working file repository,
82      // this strategy leads to a minor overhead because the file not only gets downloaded and stored in the file system
83      // but also a hard link needs to be created (or if that's not possible, a copy of the file.
84      final File origin = getUniqueFileFromWorkspace(source);
85      final File destination = createFile(storagePath, source);
86      try {
87        mkParent(destination);
88        link(origin, destination);
89      } catch (IOException e) {
90        logger.error("Error while linking/copying file {} to {}: {}", origin, destination, getMessage(e));
91        throw new AssetStoreException(e);
92      } finally {
93        if (origin != null) {
94          FileUtils.deleteQuietly(origin);
95        }
96      }
97    }
98  
99    private File getUniqueFileFromWorkspace(Source source) {
100     try {
101       return getWorkspace().get(source.getUri(), true);
102     } catch (NotFoundException e) {
103       logger.error("Source file '{}' does not exist", source.getUri());
104       throw new AssetStoreException(e);
105     } catch (IOException e) {
106       logger.error("Error while getting file '{}' from workspace: {}", source.getUri(), getMessage(e));
107       throw new AssetStoreException(e);
108     }
109   }
110 
111   @Override
112   public boolean copy(final StoragePath from, final StoragePath to) throws AssetStoreException {
113     var file = findStoragePathFile(from);
114     if (file.isPresent()) {
115       var f = file.get();
116 
117       final File t = createFile(to, f);
118       mkParent(t);
119       logger.debug("Copying {} to {}", f.getAbsolutePath(), t.getAbsolutePath());
120       try {
121         link(f, t, true);
122       } catch (IOException e) {
123         logger.error("Error copying archive file {} to {}", f, t);
124         throw new AssetStoreException(e);
125       }
126       return true;
127     }
128     return false;
129   }
130 
131   @Override
132   public Optional<InputStream> get(final StoragePath path) throws AssetStoreException {
133     var file = findStoragePathFile(path);
134     if (file.isPresent()) {
135       try {
136         return Optional.of(new FileInputStream(file.get()));
137       } catch (FileNotFoundException e) {
138         logger.error("Error getting archive file {}", file);
139         throw new AssetStoreException(e);
140       }
141     }
142     return Optional.empty();
143   }
144 
145   @Override
146   public boolean contains(StoragePath path) throws AssetStoreException {
147     return findStoragePathFile(path).isPresent();
148   }
149 
150   @Override
151   public boolean delete(DeletionSelector sel) throws AssetStoreException {
152     File dir = getDeletionSelectorDir(sel);
153     if (dir == null) {
154       // MediaPackage could not be found locally. This could mean
155       //   - all snapshots live in a remote asset store
156       //   - mount failed and files are temporary not available
157       //   - file was deleted out-of-band
158       //   - other fs problem
159       // In any case, we cannot continue and return "false" to indicate that the files could not be found.
160       return false;
161     }
162     try {
163       FileUtils.deleteDirectory(dir);
164       // also delete the media package directory if all versions have been deleted
165       boolean mpDirDeleted = FileSupport.deleteHierarchyIfEmpty(file(path(
166               getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId()), sel.getOrganizationId())),
167               dir.getParentFile());
168       if (mpDirDeleted) {
169         onDeleteMediaPackage(sel.getOrganizationId(), sel.getMediaPackageId());
170       }
171       return true;
172     } catch (IOException e) {
173       logger.error("Error deleting directory from archive {}", dir);
174       throw new AssetStoreException(e);
175     }
176   }
177 
178   /**
179    * Returns the directory file from a deletion selector
180    *
181    * @param sel
182    *          the deletion selector
183    * @return the directory file or null if it does not exist (e.g. MediaPackage does not exist locally)
184    */
185   private File getDeletionSelectorDir(DeletionSelector sel) {
186     final String rootPath = getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId());
187     if (rootPath == null) {
188       return null;
189     }
190     final String basePath = path(rootPath, sel.getOrganizationId(), sel.getMediaPackageId());
191     if (sel.getVersion().isPresent()) {
192       return file(basePath, sel.getVersion().get().toString());
193     }
194     return file(basePath);
195   }
196 
197   /** Create all parent directories of a file. */
198   private void mkParent(File f) {
199     mkDirs(f.getParentFile());
200   }
201 
202   /** Create this directory and all of its parents. */
203   protected void mkDirs(File d) {
204     if (d == null) {
205       return;
206     }
207     // mkdirs *may* not have succeeded if it returns false, so check if the directory exists in that case
208     if (!d.mkdirs() && !d.exists()) {
209       final String msg = "Cannot create directory " + d;
210       logger.error(msg);
211       throw new AssetStoreException(msg);
212     }
213   }
214 
215   /** Return the extension of a file. */
216   private Opt<String> extension(File f) {
217     return trimToNone(getExtension(f.getAbsolutePath())).toOpt();
218   }
219 
220   /** Return the extension of a URI, i.e. the extension of its path. */
221   private Opt<String> extension(URI uri) {
222     try {
223       return trimToNone(getExtension(uri.toURL().getPath())).toOpt();
224     } catch (MalformedURLException e) {
225       throw new Error(e);
226     }
227   }
228 
229   /** Create a file from a storage path and the extension of file <code>f</code>. */
230   private File createFile(StoragePath p, File f) {
231     return createFile(p, extension(f));
232   }
233 
234   /** Create a file from a storage path and the extension of the URI of <code>s</code>. */
235   private File createFile(StoragePath p, Source s) {
236     return createFile(p, extension(s.getUri()));
237   }
238 
239   /** Create a file from a storage path and an optional extension. */
240   private File createFile(StoragePath p, Opt<String> extension) {
241     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
242     if (rootDirectory == null) {
243       rootDirectory = getRootDirectory();
244     }
245     return file(
246             rootDirectory,
247             p.getOrganizationId(),
248             p.getMediaPackageId(),
249             p.getVersion().toString(),
250             extension.isSome() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
251                     .getMediaPackageElementId());
252   }
253 
254   /** Returns a file from a storage path if it exists, null otherwise */
255   private File getExistingFile(StoragePath p, Opt<String> extension) {
256     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
257     if (rootDirectory == null) {
258       return null;
259     }
260     return file(
261             rootDirectory,
262             p.getOrganizationId(),
263             p.getMediaPackageId(),
264             p.getVersion().toString(),
265             extension.isSome() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
266                     .getMediaPackageElementId());
267   }
268 
269   /**
270    * Returns a file {@link Option} from a storage path if one is found or an empty {@link Option}
271    *
272    * @param storagePath
273    *          the storage path
274    * @return the file {@link Option}
275    */
276   private Optional<File> findStoragePathFile(final StoragePath storagePath) {
277     final FilenameFilter filter = new FilenameFilter() {
278       @Override
279       public boolean accept(File dir, String name) {
280         return FilenameUtils.getBaseName(name).equals(storagePath.getMediaPackageElementId());
281       }
282     };
283     final File containerDir = getExistingFile(storagePath, Opt.none(String.class)).getParentFile();
284 
285     var files = containerDir.listFiles(filter);
286     if (files == null) {
287       return Optional.empty();
288     }
289     switch (files.length) {
290       case 0:
291         return Optional.empty();
292       case 1:
293         return Optional.of(files[0]);
294       default:
295         throw new AssetStoreException("Storage path " + files[0].getParent()
296             + "contains multiple files with the same element id!: " + storagePath.getMediaPackageElementId());
297     }
298   }
299 
300   @Override
301   public String getStoreType() {
302     return storeType;
303   }
304 
305 }