View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.storage.impl.fs;
22  
23  import static org.opencastproject.util.IoSupport.file;
24  
25  import org.opencastproject.assetmanager.api.storage.AssetStore;
26  import org.opencastproject.workspace.api.Workspace;
27  
28  import com.google.common.cache.CacheBuilder;
29  import com.google.common.cache.CacheLoader;
30  import com.google.common.cache.LoadingCache;
31  import com.google.common.util.concurrent.ExecutionError;
32  import com.google.common.util.concurrent.UncheckedExecutionException;
33  
34  import org.apache.commons.io.FileUtils;
35  import org.apache.commons.lang3.StringUtils;
36  import org.osgi.service.component.ComponentContext;
37  import org.osgi.service.component.annotations.Activate;
38  import org.osgi.service.component.annotations.Component;
39  import org.osgi.service.component.annotations.Reference;
40  import org.slf4j.Logger;
41  import org.slf4j.LoggerFactory;
42  
43  import java.io.File;
44  import java.io.IOException;
45  import java.nio.file.Files;
46  import java.nio.file.Path;
47  import java.nio.file.Paths;
48  import java.util.ArrayList;
49  import java.util.Collections;
50  import java.util.List;
51  import java.util.Optional;
52  import java.util.concurrent.TimeUnit;
53  
54  import javax.naming.ConfigurationException;
55  
56  @Component(
57      property = {
58      "service.description=File system based asset store",
59      "store.type=local-filesystem"
60      },
61      immediate = true,
62      service = { AssetStore.class }
63  )
64  public class OsgiFileSystemAssetStore extends AbstractFileSystemAssetStore {
65    /** Log facility */
66    private static final Logger logger = LoggerFactory.getLogger(OsgiFileSystemAssetStore.class);
67  
68    /** A cache of mediapckage ids and their associated storages */
69    private LoadingCache<String, Optional<String>> cache = null;
70    private int cacheSize = 1000;
71    private int cacheExpiration = 1;
72  
73    /** Configuration key for the default Opencast storage directory. A value is optional. */
74    public static final String CFG_OPT_STORAGE_DIR = "org.opencastproject.storage.dir";
75  
76    /**
77     * The default store directory name.
78     * Will be used in conjunction with {@link #CFG_OPT_STORAGE_DIR} if {@link #CFG_OPT_STORAGE_DIR} is not set.
79     */
80    private static final String DEFAULT_STORE_DIRECTORY = "archive";
81  
82    /** Configuration key for the archive root directory. */
83    public static final String CONFIG_STORE_ROOT_DIR = "org.opencastproject.episode.rootdir";
84  
85    /** The root directories for storing files (typically one) */
86    private List<String> rootDirectories;
87  
88    /** The workspace */
89    private Workspace workspace;
90  
91    @Override protected Workspace getWorkspace() {
92      return workspace;
93    }
94  
95    @Override
96    /**
97     * Returns the root directory with the most usable space left
98     * @return The root directory path
99     */
100   protected String getRootDirectory() {
101     // Determine which storage to return by amount of remaining usable space
102     long usableSpace = 0;
103     String mostUsableDirectory = null;
104     for (String path : rootDirectories) {
105       Optional<Long> maybeUsableSpace = Optional.of(new File(path).getUsableSpace());
106       if (maybeUsableSpace.isEmpty()) {
107         continue;
108       }
109       if (maybeUsableSpace.get() > usableSpace) {
110         usableSpace = maybeUsableSpace.get();
111         mostUsableDirectory = path;
112       }
113     }
114 
115     return mostUsableDirectory;
116   }
117 
118   /**
119    * Looks for the root directory of the given mediapackage id
120    * @param orgId the organization which the mediapackage belongs to
121    * @param mpId the mediapackage id
122    * @return The root directory path of the given mediapackage, or null if the mediapackage could not be found anywhere
123    */
124   protected String getRootDirectory(String orgId, String mpId) {
125     try {
126       String cacheKey = Paths.get(orgId, mpId).toString();
127       Optional<String> pathOpt = cache.getUnchecked(cacheKey);
128       if (pathOpt.isPresent()) {
129         logger.debug("Root directory for mediapackage {} is {}", mpId, pathOpt.get());
130         return pathOpt.get();
131       } else {
132         logger.debug("Root directory for mediapackage {} could not be found, returning null.", mpId);
133         cache.invalidate(cacheKey);
134         return null;
135       }
136     } catch (ExecutionError e) {
137       logger.warn("Exception while getting path for mediapackage {}", mpId, e);
138       return null;
139     } catch (UncheckedExecutionException e) {
140       logger.warn("Exception while getting path for  mediapackage {}", mpId, e);
141       return null;
142     }
143   }
144 
145   /**
146    * Looks for the root directory that contains the given mediapackage id.
147    * Used by the cache.
148    * @param orgAndMpId The part of the path that contains the organization id and mediapacakge id
149    * @return The root directory path of the given mediapackage
150    */
151   private String getRootDirectoryForMediaPackage(String orgAndMpId) {
152     // Search the mediapackage on all storages
153     for (String path : rootDirectories) {
154       Path dirPath = Path.of(path, orgAndMpId);
155       if (Files.exists(dirPath) && Files.isDirectory(dirPath)) {
156         return path;
157       }
158     }
159 
160     return null;
161   }
162 
163   private List<String> getRootDirectories() {
164     return Collections.unmodifiableList(rootDirectories);
165   }
166 
167   protected void setupCache() {
168     cache = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterWrite(cacheExpiration, TimeUnit.MINUTES)
169             .build(new CacheLoader<String, Optional<String>>() {
170               @Override
171               public Optional<String> load(String orgAndMpId) throws Exception {
172                 String rootDirectory = getRootDirectoryForMediaPackage(orgAndMpId);
173                 return rootDirectory == null ? Optional.empty() : Optional.of(rootDirectory);
174               }
175             });
176   }
177 
178   protected void onDeleteMediaPackage(String orgId, String mpId) {
179     String cacheKey = Paths.get(orgId, mpId).toString();
180     cache.invalidate(cacheKey);
181   }
182 
183   /**
184    * OSGi DI.
185    */
186   @Reference
187   public void setWorkspace(Workspace workspace) {
188     this.workspace = workspace;
189   }
190 
191   /**
192    * Service activator, called via declarative services configuration.
193    *
194    * @param cc
195    *          the component context
196    */
197   @Activate
198   public void activate(final ComponentContext cc) throws IllegalStateException, IOException, ConfigurationException {
199     storeType = (String) cc.getProperties().get(AssetStore.STORE_TYPE_PROPERTY);
200     logger.info("{} is: {}", AssetStore.STORE_TYPE_PROPERTY, storeType);
201 
202     rootDirectories = new ArrayList<>();
203 
204     // Read in single directory
205     String rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR));
206     if (rootDirectory == null) {
207       final String storageDir = StringUtils.trimToNull(cc.getBundleContext().getProperty(CFG_OPT_STORAGE_DIR));
208       if (storageDir == null) {
209         throw new IllegalArgumentException("Storage directory must be set");
210       }
211       rootDirectory = Paths.get(storageDir, DEFAULT_STORE_DIRECTORY).toFile().getAbsolutePath();
212     }
213     mkDirs(file(rootDirectory));
214     rootDirectories.add(rootDirectory);
215 
216     // Read in multiple directories
217     int index = 1;
218     boolean isRootDirectory = true;
219     while (isRootDirectory) {
220       String directory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR + "." + index));
221 
222       if (directory != null) {
223         rootDirectories.add(directory);
224       } else {
225         isRootDirectory = false;
226       }
227       index++;
228     }
229     // Check for bad configuration
230     for (int i = 0; i < rootDirectories.size(); i++) {
231       for (int j = 0; j < rootDirectories.size(); j++) {
232         if (i == j) {
233           continue;
234         }
235         if (isChild(rootDirectories.get(j), rootDirectories.get(i))) {
236           throw new ConfigurationException("Storage directory " + rootDirectories.get(j) + " is a subdirectory of "
237               + rootDirectories.get(i) + ". This is not allowed.");
238         }
239       }
240     }
241     // Create
242     for (String directory: rootDirectories) {
243       mkDirs(file(directory));
244     }
245     // Check for write access
246     for (String directory : rootDirectories) {
247       File tmp = new File(directory + "/tobedeleted.tmp");
248       tmp.createNewFile();
249       tmp.delete();
250     }
251 
252     logger.info("Start asset manager files system store at {}", rootDirectories);
253 
254     // Setup rootDirectory cache
255     // Remembers the root directory for a given mediapackage
256     setupCache();
257   }
258 
259   private static boolean isChild(String childText, String parentText) {
260     Path parent = Paths.get(parentText).toAbsolutePath();
261     Path child = Paths.get(childText).toAbsolutePath();
262     if (child.startsWith(parent)) {
263       return true;
264     }
265     return false;
266   }
267 
268   // Depending on how these functions are used, it may not make sense to just sum over all root directories.
269   // It would likely be more proper to return the individual values for each directory in a collection.
270   // However, that would require a major rewrite of the StorageUsage interface, which is a lot of work for some
271   // functions that seem to see no use anyhow.
272   @Override
273   public Optional<Long> getUsedSpace() {
274     long usedSpace = 0;
275     for (String path : rootDirectories) {
276       usedSpace += FileUtils.sizeOfDirectory(new File(path));
277     }
278     return Optional.of(usedSpace);
279   }
280 
281   @Override
282   public Optional<Long> getUsableSpace() {
283     long usableSpace = 0;
284     for (String path : rootDirectories) {
285       usableSpace += new File(path).getUsableSpace();
286     }
287     return Optional.of(usableSpace);
288   }
289 
290   @Override
291   public Optional<Long> getTotalSpace() {
292     long totalSpace = 0;
293     for (String path : rootDirectories) {
294       totalSpace += new File(path).getTotalSpace();
295     }
296     return Optional.of(totalSpace);
297   }
298 
299 }