View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  
22  package org.opencastproject.staticfiles.impl;
23  
24  import static java.lang.String.format;
25  import static org.opencastproject.util.RequireUtil.notNull;
26  
27  import org.opencastproject.security.api.Organization;
28  import org.opencastproject.security.api.OrganizationDirectoryService;
29  import org.opencastproject.security.api.SecurityService;
30  import org.opencastproject.staticfiles.api.StaticFileService;
31  import org.opencastproject.staticfiles.jmx.UploadStatistics;
32  import org.opencastproject.util.NotFoundException;
33  import org.opencastproject.util.OsgiUtil;
34  import org.opencastproject.util.ProgressInputStream;
35  import org.opencastproject.util.jmx.JmxUtil;
36  
37  import com.google.common.util.concurrent.AbstractScheduledService;
38  import com.google.common.util.concurrent.MoreExecutors;
39  import com.google.common.util.concurrent.Service.Listener;
40  import com.google.common.util.concurrent.Service.State;
41  
42  import org.apache.commons.io.FileUtils;
43  import org.apache.commons.lang3.StringUtils;
44  import org.osgi.service.component.ComponentContext;
45  import org.osgi.service.component.ComponentException;
46  import org.osgi.service.component.annotations.Activate;
47  import org.osgi.service.component.annotations.Component;
48  import org.osgi.service.component.annotations.Deactivate;
49  import org.osgi.service.component.annotations.Reference;
50  import org.slf4j.Logger;
51  import org.slf4j.LoggerFactory;
52  
53  import java.beans.PropertyChangeEvent;
54  import java.beans.PropertyChangeListener;
55  import java.io.File;
56  import java.io.IOException;
57  import java.io.InputStream;
58  import java.nio.file.DirectoryStream;
59  import java.nio.file.Files;
60  import java.nio.file.Path;
61  import java.nio.file.Paths;
62  import java.util.Date;
63  import java.util.UUID;
64  import java.util.concurrent.TimeUnit;
65  
66  import javax.management.ObjectInstance;
67  
68  /**
69   * Stores and retrieves static file resources.
70   */
71  @Component(
72      immediate = true,
73      service = StaticFileService.class,
74      property = {
75          "service.description=Static File Service",
76          "service.PID=org.opencastproject.staticfiles.impl.StaticFileServiceImpl"
77      }
78  )
79  public class StaticFileServiceImpl implements StaticFileService {
80  
81    /** The logger */
82    private static final Logger logger = LoggerFactory.getLogger(StaticFileServiceImpl.class);
83  
84    /** The key to find the root directory for the static file service in the OSGi properties. */
85    public static final String STATICFILES_ROOT_DIRECTORY_KEY = "org.opencastproject.staticfiles.rootdir";
86  
87    /** The JMX business object for uploaded statistics */
88    private UploadStatistics staticFileStatistics = new UploadStatistics();
89  
90    /** The JMX bean object instance */
91    private ObjectInstance registerMXBean;
92  
93    // OSGi service references
94    private SecurityService securityService = null;
95    private OrganizationDirectoryService orgDirectory = null;
96  
97    /** The root directory for storing static files. */
98    private String rootDirPath;
99  
100   private PurgeTemporaryStorageService purgeService;
101 
102   /**
103    * OSGI callback for activating this component
104    *
105    * @param cc
106    *          the osgi component context
107    */
108   @Activate
109   public void activate(ComponentContext cc) {
110     logger.info("Upload Static Resource Service started.");
111     registerMXBean = JmxUtil.registerMXBean(staticFileStatistics, "UploadStatistics");
112     rootDirPath = OsgiUtil.getContextProperty(cc, STATICFILES_ROOT_DIRECTORY_KEY);
113 
114     final File rootFile = new File(rootDirPath);
115     if (!rootFile.exists()) {
116       try {
117         FileUtils.forceMkdir(rootFile);
118       } catch (IOException e) {
119         throw new ComponentException(
120                 String.format("%s does not exists and could not be created", rootFile.getAbsolutePath()));
121       }
122     }
123     if (!rootFile.canRead()) {
124       throw new ComponentException(String.format("Cannot read from %s", rootFile.getAbsolutePath()));
125     }
126 
127     purgeService = new PurgeTemporaryStorageService();
128     purgeService.addListener(new Listener() {
129       @Override
130       public void failed(State from, Throwable failure) {
131         logger.warn("Temporary storage purging service failed:", failure);
132       }
133     }, MoreExecutors.directExecutor());
134     purgeService.startAsync();
135     logger.info("Purging of temporary storage section scheduled");
136   }
137 
138   /**
139    * Callback from OSGi on service deactivation.
140    */
141   @Deactivate
142   public void deactivate() {
143     JmxUtil.unregisterMXBean(registerMXBean);
144 
145     purgeService.stopAsync();
146     purgeService = null;
147   }
148 
149   /** OSGi DI */
150   @Reference
151   public void setSecurityService(SecurityService securityService) {
152     this.securityService = securityService;
153   }
154 
155   /** OSGi DI */
156   @Reference
157   public void setOrganizationDirectoryService(OrganizationDirectoryService directoryService) {
158     orgDirectory = directoryService;
159   }
160 
161   @Override
162   public String storeFile(String filename, InputStream inputStream) throws IOException {
163     notNull(filename, "filename");
164     notNull(inputStream, "inputStream");
165     final String uuid = UUID.randomUUID().toString();
166     final String org = securityService.getOrganization().getId();
167 
168     Path file = getTemporaryStorageDir(org).resolve(Paths.get(uuid, filename));
169     try (ProgressInputStream progressInputStream = new ProgressInputStream(inputStream)) {
170       progressInputStream.addPropertyChangeListener(new PropertyChangeListener() {
171         @Override
172         public void propertyChange(PropertyChangeEvent evt) {
173           long totalNumBytesRead = (Long) evt.getNewValue();
174           long oldTotalNumBytesRead = (Long) evt.getOldValue();
175           staticFileStatistics.add(totalNumBytesRead - oldTotalNumBytesRead);
176         }
177       });
178 
179       Files.createDirectories(file.getParent());
180       Files.copy(progressInputStream, file);
181     } catch (IOException e) {
182       logger.error("Unable to save file '{}' to {}", filename, file, e);
183       throw e;
184     }
185 
186     return uuid;
187   }
188 
189   @Override
190   public InputStream getFile(final String uuid) throws NotFoundException, IOException {
191     if (StringUtils.isBlank(uuid)) {
192       throw new IllegalArgumentException("The uuid must not be blank");
193     }
194 
195     final String org = securityService.getOrganization().getId();
196 
197     return Files.newInputStream(getFile(org, uuid));
198   }
199 
200   @Override
201   public void persistFile(final String uuid) throws NotFoundException, IOException {
202     final String org = securityService.getOrganization().getId();
203     try (DirectoryStream<Path> folders = Files.newDirectoryStream(getTemporaryStorageDir(org),
204             getDirsEqualsUuidFilter(uuid))) {
205       for (Path folder : folders) {
206         Files.move(folder, getDurableStorageDir(org).resolve(folder.getFileName()));
207       }
208     }
209   }
210 
211   @Override
212   public void deleteFile(String uuid) throws NotFoundException, IOException {
213     final String org = securityService.getOrganization().getId();
214     Path file = getFile(org, uuid);
215     Files.deleteIfExists(file);
216   }
217 
218   @Override
219   public String getFileName(String uuid) throws NotFoundException {
220     final String org = securityService.getOrganization().getId();
221     try {
222       Path file = getFile(org, uuid);
223       return file.getFileName().toString();
224     } catch (IOException e) {
225       logger.warn("Error while reading file:", e);
226       throw new NotFoundException(e);
227     }
228   }
229 
230   @Override
231   public Long getContentLength(String uuid) throws NotFoundException {
232     final String org = securityService.getOrganization().getId();
233     try {
234       Path file = getFile(org, uuid);
235       return Files.size(file);
236     } catch (IOException e) {
237       logger.warn("Error while reading file:", e);
238       throw new NotFoundException(e);
239     }
240   }
241 
242   /**
243    * Returns a {@link DirectoryStream.Filter} to filter the entries of a directory and only return items which filename
244    * starts with the UUID.
245    *
246    * @param uuid
247    *          The UUID to filter by
248    * @return the filter
249    */
250   private static DirectoryStream.Filter<Path> getDirsEqualsUuidFilter(final String uuid) {
251     return new DirectoryStream.Filter<Path>() {
252       @Override
253       public boolean accept(Path entry) throws IOException {
254         return Files.isDirectory(entry) && entry.getFileName().toString().equals(uuid);
255       }
256     };
257   };
258 
259   /**
260    * Returns the temporary storage directory for an organization.
261    *
262    * @param org
263    *          The organization
264    * @return Path to the temporary storage directory
265    */
266   private Path getTemporaryStorageDir(final String org) {
267     return Paths.get(rootDirPath, org, "temp");
268   }
269 
270   private Path getDurableStorageDir(final String org) {
271     return Paths.get(rootDirPath, org);
272   }
273 
274   private Path getFile(final String org, final String uuid) throws NotFoundException, IOException {
275     // First check if the file is part of the durable storage section
276     try (DirectoryStream<Path> dirs = Files.newDirectoryStream(getDurableStorageDir(org),
277             getDirsEqualsUuidFilter(uuid))) {
278       for (Path dir : dirs) {
279         try (DirectoryStream<Path> files = Files.newDirectoryStream(dir)) {
280           for (Path file : files) {
281             return file;
282           }
283         }
284       }
285     }
286 
287     // Second check if the file is part of the temporary storage section
288     try (DirectoryStream<Path> dirs = Files.newDirectoryStream(getTemporaryStorageDir(org),
289             getDirsEqualsUuidFilter(uuid))) {
290       for (Path dir : dirs) {
291         try (DirectoryStream<Path> files = Files.newDirectoryStream(dir)) {
292           for (Path file : files) {
293             return file;
294           }
295         }
296       }
297     }
298 
299     throw new NotFoundException(format("No file with UUID '%s' found.", uuid));
300   }
301 
302   /**
303    * Deletes all files found in the temporary storage section of an organization.
304    *
305    * @param org
306    *          The organization identifier
307    * @throws IOException
308    *           if there was an error while deleting the files.
309    */
310   void purgeTemporaryStorageSection(final String org, final long lifetime) throws IOException {
311     logger.debug("Purge temporary storage section of organization '{}'", org);
312     final Path temporaryStorageDir = getTemporaryStorageDir(org);
313     if (Files.exists(temporaryStorageDir)) {
314       try (DirectoryStream<Path> tempFilesStream = Files.newDirectoryStream(temporaryStorageDir,
315           new DirectoryStream.Filter<Path>() {
316             @Override
317             public boolean accept(Path path) throws IOException {
318               return (Files.getLastModifiedTime(path).toMillis() < (new Date()).getTime() - lifetime);
319             }
320           })) {
321         for (Path file : tempFilesStream) {
322           FileUtils.deleteQuietly(file.toFile());
323         }
324       }
325     }
326   }
327 
328   /**
329    * Deletes all files found in the temporary storage section of all known organizations.
330    *
331    * @throws IOException
332    *           if there was an error while deleting the files.
333    */
334   void purgeTemporaryStorageSection() throws IOException {
335     logger.info("Start purging temporary storage section of all known organizations");
336     for (Organization org : orgDirectory.getOrganizations()) {
337       purgeTemporaryStorageSection(org.getId(), TimeUnit.DAYS.toMillis(1));
338     }
339   }
340 
341   /** Scheduled service for purging temporary storage sections. */
342   private class PurgeTemporaryStorageService extends AbstractScheduledService {
343 
344     @Override
345     protected void runOneIteration() throws Exception {
346       StaticFileServiceImpl.this.purgeTemporaryStorageSection();
347     }
348 
349     @Override
350     protected Scheduler scheduler() {
351       return Scheduler.newFixedRateSchedule(0, 1, TimeUnit.HOURS);
352     }
353 
354   }
355 
356 }