View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.storage.impl.fs;
22  
23  import static org.apache.commons.io.FilenameUtils.EXTENSION_SEPARATOR;
24  import static org.apache.commons.io.FilenameUtils.getExtension;
25  import static org.apache.commons.lang3.exception.ExceptionUtils.getMessage;
26  import static org.opencastproject.util.FileSupport.link;
27  import static org.opencastproject.util.IoSupport.file;
28  import static org.opencastproject.util.PathSupport.path;
29  import static org.opencastproject.util.data.functions.Strings.trimToNone;
30  
31  import org.opencastproject.assetmanager.api.storage.AssetStore;
32  import org.opencastproject.assetmanager.api.storage.AssetStoreException;
33  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
34  import org.opencastproject.assetmanager.api.storage.Source;
35  import org.opencastproject.assetmanager.api.storage.StoragePath;
36  import org.opencastproject.util.FileSupport;
37  import org.opencastproject.util.NotFoundException;
38  import org.opencastproject.util.data.Option;
39  import org.opencastproject.workspace.api.Workspace;
40  
41  import org.apache.commons.io.FileUtils;
42  import org.apache.commons.io.FilenameUtils;
43  import org.slf4j.Logger;
44  import org.slf4j.LoggerFactory;
45  
46  import java.io.File;
47  import java.io.FileInputStream;
48  import java.io.FileNotFoundException;
49  import java.io.FilenameFilter;
50  import java.io.IOException;
51  import java.io.InputStream;
52  import java.net.MalformedURLException;
53  import java.net.URI;
54  import java.util.Optional;
55  
56  public abstract class AbstractFileSystemAssetStore implements AssetStore {
57    /** Log facility */
58    private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystemAssetStore.class);
59  
60    /** The store type e.g. filesystem (short-term), aws (long-term), other implementations */
61    protected String storeType = null;
62  
63    protected abstract Workspace getWorkspace();
64  
65    protected abstract String getRootDirectory();
66    protected abstract String getRootDirectory(String orgId, String mpId);
67  
68    /**
69     * Optional further handling of the complete deletion of mediapackage from the local store.
70     * This method will be called after the deletion of the mediapackage directory.
71     * @param orgId Organization ID
72     * @param mpId Mediapackage ID
73     */
74    protected abstract void onDeleteMediaPackage(String orgId, String mpId);
75  
76    @Override
77    public void put(StoragePath storagePath, Source source) throws AssetStoreException {
78      // Retrieving the file from the workspace has the advantage that in most cases the file already exists in the local
79      // working file repository. In the very few cases where the file is not in the working file repository,
80      // this strategy leads to a minor overhead because the file not only gets downloaded and stored in the file system
81      // but also a hard link needs to be created (or if that's not possible, a copy of the file.
82      final File origin = getUniqueFileFromWorkspace(source);
83      final File destination = createFile(storagePath, source);
84      try {
85        mkParent(destination);
86        link(origin, destination);
87      } catch (IOException e) {
88        logger.error("Error while linking/copying file {} to {}: {}", origin, destination, getMessage(e));
89        throw new AssetStoreException(e);
90      } finally {
91        if (origin != null) {
92          FileUtils.deleteQuietly(origin);
93        }
94      }
95    }
96  
97    private File getUniqueFileFromWorkspace(Source source) {
98      try {
99        return getWorkspace().get(source.getUri(), true);
100     } catch (NotFoundException e) {
101       logger.error("Source file '{}' does not exist", source.getUri());
102       throw new AssetStoreException(e);
103     } catch (IOException e) {
104       logger.error("Error while getting file '{}' from workspace: {}", source.getUri(), getMessage(e));
105       throw new AssetStoreException(e);
106     }
107   }
108 
109   @Override
110   public boolean copy(final StoragePath from, final StoragePath to) throws AssetStoreException {
111     var file = findStoragePathFile(from);
112     if (file.isPresent()) {
113       var f = file.get();
114 
115       final File t = createFile(to, f);
116       mkParent(t);
117       logger.debug("Copying {} to {}", f.getAbsolutePath(), t.getAbsolutePath());
118       try {
119         link(f, t, true);
120       } catch (IOException e) {
121         logger.error("Error copying archive file {} to {}", f, t);
122         throw new AssetStoreException(e);
123       }
124       return true;
125     }
126     return false;
127   }
128 
129   @Override
130   public Optional<InputStream> get(final StoragePath path) throws AssetStoreException {
131     var file = findStoragePathFile(path);
132     if (file.isPresent()) {
133       try {
134         return Optional.of(new FileInputStream(file.get()));
135       } catch (FileNotFoundException e) {
136         logger.error("Error getting archive file {}", file);
137         throw new AssetStoreException(e);
138       }
139     }
140     return Optional.empty();
141   }
142 
143   @Override
144   public boolean contains(StoragePath path) throws AssetStoreException {
145     return findStoragePathFile(path).isPresent();
146   }
147 
148   @Override
149   public boolean delete(DeletionSelector sel) throws AssetStoreException {
150     File dir = getDeletionSelectorDir(sel);
151     if (dir == null) {
152       // MediaPackage could not be found locally. This could mean
153       //   - all snapshots live in a remote asset store
154       //   - mount failed and files are temporary not available
155       //   - file was deleted out-of-band
156       //   - other fs problem
157       // In any case, we cannot continue and return "false" to indicate that the files could not be found.
158       return false;
159     }
160     try {
161       FileUtils.deleteDirectory(dir);
162       // also delete the media package directory if all versions have been deleted
163       boolean mpDirDeleted = FileSupport.deleteHierarchyIfEmpty(file(path(
164               getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId()), sel.getOrganizationId())),
165               dir.getParentFile());
166       if (mpDirDeleted) {
167         onDeleteMediaPackage(sel.getOrganizationId(), sel.getMediaPackageId());
168       }
169       return true;
170     } catch (IOException e) {
171       logger.error("Error deleting directory from archive {}", dir);
172       throw new AssetStoreException(e);
173     }
174   }
175 
176   /**
177    * Returns the directory file from a deletion selector
178    *
179    * @param sel
180    *          the deletion selector
181    * @return the directory file or null if it does not exist (e.g. MediaPackage does not exist locally)
182    */
183   private File getDeletionSelectorDir(DeletionSelector sel) {
184     final String rootPath = getRootDirectory(sel.getOrganizationId(), sel.getMediaPackageId());
185     if (rootPath == null) {
186       return null;
187     }
188     final String basePath = path(rootPath, sel.getOrganizationId(), sel.getMediaPackageId());
189     if (sel.getVersion().isPresent()) {
190       return file(basePath, sel.getVersion().get().toString());
191     }
192     return file(basePath);
193   }
194 
195   /** Create all parent directories of a file. */
196   private void mkParent(File f) {
197     mkDirs(f.getParentFile());
198   }
199 
200   /** Create this directory and all of its parents. */
201   protected void mkDirs(File d) {
202     if (d == null) {
203       return;
204     }
205     // mkdirs *may* not have succeeded if it returns false, so check if the directory exists in that case
206     if (!d.mkdirs() && !d.exists()) {
207       final String msg = "Cannot create directory " + d;
208       logger.error(msg);
209       throw new AssetStoreException(msg);
210     }
211   }
212 
213   /** Return the extension of a file. */
214   private Optional<String> extension(File f) {
215     Option<String> opt = trimToNone(getExtension(f.getAbsolutePath()));
216     return opt.isSome()
217         ? Optional.of(opt.get())
218         : Optional.empty();
219   }
220 
221   /** Return the extension of a URI, i.e. the extension of its path. */
222   private Optional<String> extension(URI uri) {
223     try {
224       Option<String> opt = trimToNone(getExtension(uri.toURL().getPath()));
225       return opt.isSome()
226           ? Optional.of(opt.get())
227           : Optional.empty();
228     } catch (MalformedURLException e) {
229       throw new Error(e);
230     }
231   }
232 
233   /** Create a file from a storage path and the extension of file <code>f</code>. */
234   private File createFile(StoragePath p, File f) {
235     return createFile(p, extension(f));
236   }
237 
238   /** Create a file from a storage path and the extension of the URI of <code>s</code>. */
239   private File createFile(StoragePath p, Source s) {
240     return createFile(p, extension(s.getUri()));
241   }
242 
243   /** Create a file from a storage path and an optional extension. */
244   private File createFile(StoragePath p, Optional<String> extension) {
245     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
246     if (rootDirectory == null) {
247       rootDirectory = getRootDirectory();
248     }
249     return file(
250             rootDirectory,
251             p.getOrganizationId(),
252             p.getMediaPackageId(),
253             p.getVersion().toString(),
254             extension.isPresent() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
255                     .getMediaPackageElementId());
256   }
257 
258   /** Returns a file from a storage path if it exists, null otherwise */
259   private File getExistingFile(StoragePath p, Optional<String> extension) {
260     String rootDirectory = getRootDirectory(p.getOrganizationId(), p.getMediaPackageId());
261     if (rootDirectory == null) {
262       return null;
263     }
264     return file(
265             rootDirectory,
266             p.getOrganizationId(),
267             p.getMediaPackageId(),
268             p.getVersion().toString(),
269             extension.isPresent() ? p.getMediaPackageElementId() + EXTENSION_SEPARATOR + extension.get() : p
270                     .getMediaPackageElementId());
271   }
272 
273   /**
274    * Returns a file {@link Option} from a storage path if one is found or an empty {@link Option}
275    *
276    * @param storagePath
277    *          the storage path
278    * @return the file {@link Option}
279    */
280   private Optional<File> findStoragePathFile(final StoragePath storagePath) {
281     final FilenameFilter filter = new FilenameFilter() {
282       @Override
283       public boolean accept(File dir, String name) {
284         return FilenameUtils.getBaseName(name).equals(storagePath.getMediaPackageElementId());
285       }
286     };
287     final File containerDir = getExistingFile(storagePath, Optional.empty()).getParentFile();
288 
289     var files = containerDir.listFiles(filter);
290     if (files == null) {
291       return Optional.empty();
292     }
293     switch (files.length) {
294       case 0:
295         return Optional.empty();
296       case 1:
297         return Optional.of(files[0]);
298       default:
299         throw new AssetStoreException("Storage path " + files[0].getParent()
300             + "contains multiple files with the same element id!: " + storagePath.getMediaPackageElementId());
301     }
302   }
303 
304   @Override
305   public String getStoreType() {
306     return storeType;
307   }
308 
309 }