1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.assetmanager.storage.impl.fs;
22
23 import static org.opencastproject.util.IoSupport.file;
24
25 import org.opencastproject.assetmanager.api.storage.AssetStore;
26 import org.opencastproject.storage.StorageUsage;
27 import org.opencastproject.workspace.api.Workspace;
28
29 import com.google.common.cache.CacheBuilder;
30 import com.google.common.cache.CacheLoader;
31 import com.google.common.cache.LoadingCache;
32 import com.google.common.util.concurrent.ExecutionError;
33 import com.google.common.util.concurrent.UncheckedExecutionException;
34
35 import org.apache.commons.io.FileUtils;
36 import org.apache.commons.lang3.StringUtils;
37 import org.osgi.service.component.ComponentContext;
38 import org.osgi.service.component.annotations.Activate;
39 import org.osgi.service.component.annotations.Component;
40 import org.osgi.service.component.annotations.Reference;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43
44 import java.io.File;
45 import java.io.IOException;
46 import java.nio.file.Files;
47 import java.nio.file.Path;
48 import java.nio.file.Paths;
49 import java.util.ArrayList;
50 import java.util.Collections;
51 import java.util.List;
52 import java.util.Optional;
53 import java.util.concurrent.TimeUnit;
54
55 import javax.naming.ConfigurationException;
56
57 @Component(
58 property = {
59 "service.description=File system based asset store",
60 "store.type=local-filesystem"
61 },
62 immediate = true,
63 service = { AssetStore.class, StorageUsage.class }
64 )
65 public class OsgiFileSystemAssetStore extends AbstractFileSystemAssetStore {
66
67 private static final Logger logger = LoggerFactory.getLogger(OsgiFileSystemAssetStore.class);
68
69
70 private LoadingCache<String, Optional<String>> cache = null;
71 private int cacheSize = 1000;
72 private int cacheExpiration = 1;
73
74
75 public static final String CFG_OPT_STORAGE_DIR = "org.opencastproject.storage.dir";
76
77
78
79
80
81 private static final String DEFAULT_STORE_DIRECTORY = "archive";
82
83
84 public static final String CONFIG_STORE_ROOT_DIR = "org.opencastproject.episode.rootdir";
85
86
87 private List<String> rootDirectories;
88
89
90 private Workspace workspace;
91
92 @Override protected Workspace getWorkspace() {
93 return workspace;
94 }
95
96 @Override
97
98
99
100
101 protected String getRootDirectory() {
102
103 long usableSpace = 0;
104 String mostUsableDirectory = null;
105 for (String path : rootDirectories) {
106 Optional<Long> maybeUsableSpace = Optional.of(new File(path).getUsableSpace());
107 if (maybeUsableSpace.isEmpty()) {
108 continue;
109 }
110 if (maybeUsableSpace.get() > usableSpace) {
111 usableSpace = maybeUsableSpace.get();
112 mostUsableDirectory = path;
113 }
114 }
115
116 return mostUsableDirectory;
117 }
118
119
120
121
122
123
124
125 protected String getRootDirectory(String orgId, String mpId) {
126 try {
127 String cacheKey = Paths.get(orgId, mpId).toString();
128 Optional<String> pathOpt = cache.getUnchecked(cacheKey);
129 if (pathOpt.isPresent()) {
130 logger.debug("Root directory for mediapackage {} is {}", mpId, pathOpt.get());
131 return pathOpt.get();
132 } else {
133 logger.debug("Root directory for mediapackage {} could not be found, returning null.", mpId);
134 cache.invalidate(cacheKey);
135 return null;
136 }
137 } catch (ExecutionError e) {
138 logger.warn("Exception while getting path for mediapackage {}", mpId, e);
139 return null;
140 } catch (UncheckedExecutionException e) {
141 logger.warn("Exception while getting path for mediapackage {}", mpId, e);
142 return null;
143 }
144 }
145
146
147
148
149
150
151
152 private String getRootDirectoryForMediaPackage(String orgAndMpId) {
153
154 for (String path : rootDirectories) {
155 Path dirPath = Path.of(path, orgAndMpId);
156 if (Files.exists(dirPath) && Files.isDirectory(dirPath)) {
157 return path;
158 }
159 }
160
161 return null;
162 }
163
164 private List<String> getRootDirectories() {
165 return Collections.unmodifiableList(rootDirectories);
166 }
167
168 protected void setupCache() {
169 cache = CacheBuilder.newBuilder().maximumSize(cacheSize).expireAfterWrite(cacheExpiration, TimeUnit.MINUTES)
170 .build(new CacheLoader<String, Optional<String>>() {
171 @Override
172 public Optional<String> load(String orgAndMpId) throws Exception {
173 String rootDirectory = getRootDirectoryForMediaPackage(orgAndMpId);
174 return rootDirectory == null ? Optional.empty() : Optional.of(rootDirectory);
175 }
176 });
177 }
178
179 protected void onDeleteMediaPackage(String orgId, String mpId) {
180 String cacheKey = Paths.get(orgId, mpId).toString();
181 cache.invalidate(cacheKey);
182 }
183
184
185
186
187 @Reference
188 public void setWorkspace(Workspace workspace) {
189 this.workspace = workspace;
190 }
191
192
193
194
195
196
197
198 @Activate
199 public void activate(final ComponentContext cc) throws IllegalStateException, IOException, ConfigurationException {
200 storeType = (String) cc.getProperties().get(AssetStore.STORE_TYPE_PROPERTY);
201 logger.info("{} is: {}", AssetStore.STORE_TYPE_PROPERTY, storeType);
202
203 rootDirectories = new ArrayList<>();
204
205
206 String rootDirectory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR));
207 if (rootDirectory == null) {
208 final String storageDir = StringUtils.trimToNull(cc.getBundleContext().getProperty(CFG_OPT_STORAGE_DIR));
209 if (storageDir == null) {
210 throw new IllegalArgumentException("Storage directory must be set");
211 }
212 rootDirectory = Paths.get(storageDir, DEFAULT_STORE_DIRECTORY).toFile().getAbsolutePath();
213 }
214 mkDirs(file(rootDirectory));
215 rootDirectories.add(rootDirectory);
216
217
218 int index = 1;
219 boolean isRootDirectory = true;
220 while (isRootDirectory) {
221 String directory = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_STORE_ROOT_DIR + "." + index));
222
223 if (directory != null) {
224 rootDirectories.add(directory);
225 } else {
226 isRootDirectory = false;
227 }
228 index++;
229 }
230
231 for (int i = 0; i < rootDirectories.size(); i++) {
232 for (int j = 0; j < rootDirectories.size(); j++) {
233 if (i == j) {
234 continue;
235 }
236 if (isChild(rootDirectories.get(j), rootDirectories.get(i))) {
237 throw new ConfigurationException("Storage directory " + rootDirectories.get(j) + " is a subdirectory of "
238 + rootDirectories.get(i) + ". This is not allowed.");
239 }
240 }
241 }
242
243 for (String directory: rootDirectories) {
244 mkDirs(file(directory));
245 }
246
247 for (String directory : rootDirectories) {
248 File tmp = new File(directory + "/tobedeleted.tmp");
249 tmp.createNewFile();
250 tmp.delete();
251 }
252
253 logger.info("Start asset manager files system store at {}", rootDirectories);
254
255
256
257 setupCache();
258 }
259
260 private static boolean isChild(String childText, String parentText) {
261 Path parent = Paths.get(parentText).toAbsolutePath();
262 Path child = Paths.get(childText).toAbsolutePath();
263 if (child.startsWith(parent)) {
264 return true;
265 }
266 return false;
267 }
268
269
270
271
272
273 @Override
274 public Optional<Long> getUsedSpace() {
275 long usedSpace = 0;
276 for (String path : rootDirectories) {
277 usedSpace += FileUtils.sizeOfDirectory(new File(path));
278 }
279 return Optional.of(usedSpace);
280 }
281
282 @Override
283 public Optional<Long> getUsableSpace() {
284 long usableSpace = 0;
285 for (String path : rootDirectories) {
286 usableSpace += new File(path).getUsableSpace();
287 }
288 return Optional.of(usableSpace);
289 }
290
291 @Override
292 public Optional<Long> getTotalSpace() {
293 long totalSpace = 0;
294 for (String path : rootDirectories) {
295 totalSpace += new File(path).getTotalSpace();
296 }
297 return Optional.of(totalSpace);
298 }
299
300 @Override
301 public String getStorageName() {
302 return "File System Asset Store";
303 }
304 }