View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.storage.impl.fs;
22  
23  import static org.opencastproject.util.IoSupport.file;
24  
25  import org.opencastproject.assetmanager.api.storage.AssetStore;
26  import org.opencastproject.storage.StorageUsage;
27  import org.opencastproject.workspace.api.Workspace;
28  
29  import com.google.common.cache.CacheBuilder;
30  import com.google.common.cache.CacheLoader;
31  import com.google.common.cache.LoadingCache;
32  import com.google.common.util.concurrent.ExecutionError;
33  import com.google.common.util.concurrent.UncheckedExecutionException;
34  
35  import org.apache.commons.io.FileUtils;
36  import org.apache.commons.lang3.StringUtils;
37  import org.osgi.service.component.ComponentContext;
38  import org.osgi.service.component.annotations.Activate;
39  import org.osgi.service.component.annotations.Component;
40  import org.osgi.service.component.annotations.Reference;
41  import org.slf4j.Logger;
42  import org.slf4j.LoggerFactory;
43  
44  import java.io.File;
45  import java.io.IOException;
46  import java.nio.file.Files;
47  import java.nio.file.Path;
48  import java.nio.file.Paths;
49  import java.util.ArrayList;
50  import java.util.Collections;
51  import java.util.List;
52  import java.util.Optional;
53  import java.util.concurrent.TimeUnit;
54  
55  import javax.naming.ConfigurationException;
56  
57  @Component(
58      property = {
59      "service.description=File system based asset store",
60      "store.type=local-filesystem"
61      },
62      immediate = true,
63      service = { AssetStore.class, StorageUsage.class }
64  )
65  public class OsgiFileSystemAssetStore extends AbstractFileSystemAssetStore {
66    /** Log facility */
67    private static final Logger logger = LoggerFactory.getLogger(OsgiFileSystemAssetStore.class);
68  
69    /** A cache of mediapckage ids and their associated storages */
70    private LoadingCache<String, Optional<String>> cache = null;
71    private int cacheSize = 1000;
72    private int cacheExpiration = 1;
73  
74    /** Configuration key for the default Opencast storage directory. A value is optional. */
75    public static final String CFG_OPT_STORAGE_DIR = "org.opencastproject.storage.dir";
76  
77    /**
78     * The default store directory name.
79     * Will be used in conjunction with {@link #CFG_OPT_STORAGE_DIR} if {@link #CFG_OPT_STORAGE_DIR} is not set.
80     */
81    private static final String DEFAULT_STORE_DIRECTORY = "archive";
82  
83    /** Configuration key for the archive root directory. */
84    public static final String CONFIG_STORE_ROOT_DIR = "org.opencastproject.episode.rootdir";
85  
86    /** The root directories for storing files (typically one) */
87    private List<String> rootDirectories;
88  
89    /** The workspace */
90    private Workspace workspace;
91  
92    @Override protected Workspace getWorkspace() {
93      return workspace;
94    }
95  
96    @Override
97    /**
98     * Returns the root directory with the most usable space left
99     * @return The root directory path
100    */
101   protected String getRootDirectory() {
102     // Determine which storage to return by amount of remaining usable space
103     long usableSpace = 0;
104     String mostUsableDirectory = null;
105     for (String path : rootDirectories) {
106       Optional<Long> maybeUsableSpace = Optional.of(new File(path).getUsableSpace());
107       if (maybeUsableSpace.isEmpty()) {
108         continue;
109       }
110       if (maybeUsableSpace.get() > usableSpace) {
111         usableSpace = maybeUsableSpace.get();
112         mostUsableDirectory = path;
113       }
114     }
115 
116     return mostUsableDirectory;
117   }
118 
119   /**
120    * Looks for the root directory of the given mediapackage id
121    * @param orgId the organization which the mediapackage belongs to
122    * @param mpId the mediapackage id
123    * @return The root directory path of the given mediapackage, or null if the mediapackage could not be found anywhere
124    */
125   protected String getRootDirectory(String orgId, String mpId) {
126     try {
127       String cacheKey = Paths.get(orgId, mpId).toString();
128       Optional<String> pathOpt = cache.getUnchecked(cacheKey);
129       if (pathOpt.isPresent()) {
130         logger.debug("Root directory for mediapackage {} is {}", mpId, pathOpt.get());
131         return pathOpt.get();
132       } else {
133         logger.debug("Root directory for mediapackage {} could not be found, returning null.", mpId);
134         cache.invalidate(cacheKey);
135         return null;
136       }
137     } catch (ExecutionError e) {
138       logger.warn("Exception while getting path for mediapackage {}", mpId, e);
139       return null;
140     } catch (UncheckedExecutionException e) {
141       logger.warn("Exception while getting path for  mediapackage {}", mpId, e);
142       return null;
143     }
144   }
145 
146   /**
147    * Looks for the root directory that contains the given mediapackage id.
148    * Used by the cache.
149    * @param orgAndMpId The part of the path that contains the organization id and mediapacakge id
150    * @return The root directory path of the given mediapackage
151    */
152   private String getRootDirectoryForMediaPackage(String orgAndMpId) {
153     // Search the mediapackage on all storages
154     for (String path : rootDirectories) {
155       Path dirPath = Path.of(path, orgAndMpId);
156       if (Files.exists(dirPath) && Files.isDirectory(dirPath)) {
157         return path;
158       }
159     }
160 
161     return null;
162   }
163 
164   private List<String> getRootDirectories() {
165     return Collections.unmodifiableList(rootDirectories);
166   }
167 
168   protected void setupCache() {
169     cache = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterWrite(cacheExpiration, TimeUnit.MINUTES)
170             .build(new CacheLoader<String, Optional<String>>() {
171               @Override
172               public Optional<String> load(String orgAndMpId) throws Exception {
173                 String rootDirectory = getRootDirectoryForMediaPackage(orgAndMpId);
174                 return rootDirectory == null ? Optional.empty() : Optional.of(rootDirectory);
175               }
176             });
177   }
178 
179   protected void onDeleteMediaPackage(String orgId, String mpId) {
180     String cacheKey = Paths.get(orgId, mpId).toString();
181     cache.invalidate(cacheKey);
182   }
183 
184   /**
185    * OSGi DI.
186    */
187   @Reference
188   public void setWorkspace(Workspace workspace) {
189     this.workspace = workspace;
190   }
191 
192   /**
193    * Service activator, called via declarative services configuration.
194    *
195    * @param cc
196    *          the component context
197    */
198   @Activate
199   public void activate(final ComponentContext cc) throws IllegalStateException, IOException, ConfigurationException {
200     storeType = (String) cc.getProperties().get(AssetStore.STORE_TYPE_PROPERTY);
201     logger.info("{} is: {}", AssetStore.STORE_TYPE_PROPERTY, storeType);
202 
203     rootDirectories = new ArrayList<>();
204 
205     // Read in single directory
206     String rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR));
207     if (rootDirectory == null) {
208       final String storageDir = StringUtils.trimToNull(cc.getBundleContext().getProperty(CFG_OPT_STORAGE_DIR));
209       if (storageDir == null) {
210         throw new IllegalArgumentException("Storage directory must be set");
211       }
212       rootDirectory = Paths.get(storageDir, DEFAULT_STORE_DIRECTORY).toFile().getAbsolutePath();
213     }
214     mkDirs(file(rootDirectory));
215     rootDirectories.add(rootDirectory);
216 
217     // Read in multiple directories
218     int index = 1;
219     boolean isRootDirectory = true;
220     while (isRootDirectory) {
221       String directory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR + "." + index));
222 
223       if (directory != null) {
224         rootDirectories.add(directory);
225       } else {
226         isRootDirectory = false;
227       }
228       index++;
229     }
230     // Check for bad configuration
231     for (int i = 0; i < rootDirectories.size(); i++) {
232       for (int j = 0; j < rootDirectories.size(); j++) {
233         if (i == j) {
234           continue;
235         }
236         if (isChild(rootDirectories.get(j), rootDirectories.get(i))) {
237           throw new ConfigurationException("Storage directory " + rootDirectories.get(j) + " is a subdirectory of "
238               + rootDirectories.get(i) + ". This is not allowed.");
239         }
240       }
241     }
242     // Create
243     for (String directory: rootDirectories) {
244       mkDirs(file(directory));
245     }
246     // Check for write access
247     for (String directory : rootDirectories) {
248       File tmp = new File(directory + "/tobedeleted.tmp");
249       tmp.createNewFile();
250       tmp.delete();
251     }
252 
253     logger.info("Start asset manager files system store at {}", rootDirectories);
254 
255     // Setup rootDirectory cache
256     // Remembers the root directory for a given mediapackage
257     setupCache();
258   }
259 
260   private static boolean isChild(String childText, String parentText) {
261     Path parent = Paths.get(parentText).toAbsolutePath();
262     Path child = Paths.get(childText).toAbsolutePath();
263     if (child.startsWith(parent)) {
264       return true;
265     }
266     return false;
267   }
268 
269   // Depending on how these functions are used, it may not make sense to just sum over all root directories.
270   // It would likely be more proper to return the individual values for each directory in a collection.
271   // However, that would require a major rewrite of the StorageUsage interface, which is a lot of work for some
272   // functions that seem to see no use anyhow.
273   @Override
274   public Optional<Long> getUsedSpace() {
275     long usedSpace = 0;
276     for (String path : rootDirectories) {
277       usedSpace += FileUtils.sizeOfDirectory(new File(path));
278     }
279     return Optional.of(usedSpace);
280   }
281 
282   @Override
283   public Optional<Long> getUsableSpace() {
284     long usableSpace = 0;
285     for (String path : rootDirectories) {
286       usableSpace += new File(path).getUsableSpace();
287     }
288     return Optional.of(usableSpace);
289   }
290 
291   @Override
292   public Optional<Long> getTotalSpace() {
293     long totalSpace = 0;
294     for (String path : rootDirectories) {
295       totalSpace += new File(path).getTotalSpace();
296     }
297     return Optional.of(totalSpace);
298   }
299 
300   @Override
301   public String getStorageName() {
302     return "File System Asset Store";
303   }
304 }