View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  
22  package org.opencastproject.assetmanager.aws;
23  
24  import static org.apache.commons.lang3.exception.ExceptionUtils.getMessage;
25  
26  import org.opencastproject.assetmanager.api.storage.AssetStore;
27  import org.opencastproject.assetmanager.api.storage.AssetStoreException;
28  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
29  import org.opencastproject.assetmanager.api.storage.Source;
30  import org.opencastproject.assetmanager.api.storage.StoragePath;
31  import org.opencastproject.assetmanager.aws.persistence.AwsAssetDatabase;
32  import org.opencastproject.assetmanager.aws.persistence.AwsAssetDatabaseException;
33  import org.opencastproject.assetmanager.aws.persistence.AwsAssetMapping;
34  import org.opencastproject.assetmanager.impl.VersionImpl;
35  import org.opencastproject.util.ConfigurationException;
36  import org.opencastproject.util.MimeType;
37  import org.opencastproject.util.NotFoundException;
38  import org.opencastproject.util.OsgiUtil;
39  import org.opencastproject.util.data.Option;
40  import org.opencastproject.workspace.api.Workspace;
41  
42  import org.apache.commons.io.FilenameUtils;
43  import org.apache.commons.lang3.StringUtils;
44  import org.osgi.service.component.ComponentContext;
45  import org.slf4j.Logger;
46  import org.slf4j.LoggerFactory;
47  
48  import java.io.File;
49  import java.io.IOException;
50  import java.io.InputStream;
51  import java.util.List;
52  import java.util.Optional;
53  
54  public abstract class AwsAbstractArchive implements AssetStore {
55  
56    /** Log facility */
57    private static final Logger logger = LoggerFactory.getLogger(AwsAbstractArchive.class);
58  
59    protected Workspace workspace;
60    protected AwsAssetDatabase database;
61  
62    /** The store type e.g. aws (long-term), or other implementations */
63    protected String storeType = null;
64    /** The AWS region */
65    protected String regionName = null;
66  
67    protected String getAWSConfigKey(ComponentContext cc, String key) {
68      try {
69        String value = StringUtils.trimToEmpty(OsgiUtil.getComponentContextProperty(cc, key));
70        if (StringUtils.isNotBlank(value)) {
71          return value;
72        }
73        throw new ConfigurationException(key + " is invalid");
74      } catch (RuntimeException e) {
75        throw new ConfigurationException(key + " is missing or invalid", e);
76      }
77    }
78  
79    public Option<Long> getUsedSpace() {
80      throw new UnsupportedOperationException("Not implemented");
81    }
82  
83    public Option<Long> getUsableSpace() {
84      throw new UnsupportedOperationException("Not implemented");
85    }
86  
87    public Option<Long> getTotalSpace() {
88      throw new UnsupportedOperationException("Not implemented");
89    }
90  
91    public String getStoreType() {
92      return this.storeType;
93    }
94  
95    public String getRegion() {
96      return this.regionName;
97    }
98  
99    /** OSGi Di */
100   public void setWorkspace(Workspace workspace) {
101     this.workspace = workspace;
102   }
103 
104   /** OSGi Di */
105   public void setDatabase(AwsAssetDatabase db) {
106     this.database = db;
107   }
108 
109   /** @see AssetStore#copy(StoragePath, StoragePath) */
110   public boolean copy(final StoragePath from, final StoragePath to) throws AssetStoreException {
111     try {
112       AwsAssetMapping map = database.findMapping(from);
113       if (!contains(from)) {
114         logger.warn("Origin file mapping not found in database: {}", from);
115         return false;
116       }
117       // New mapping will point to the SAME AWS object, nothing will be uploaded
118       logger.debug("Adding AWS {} link mapping to database: {} points to {}, version {}", getStoreType(),
119               to, map.getObjectKey(), map.getObjectVersion());
120       database.storeMapping(to, map.getObjectKey(), map.getObjectVersion());
121       return true;
122     } catch (AwsAssetDatabaseException e) {
123       throw new AssetStoreException(e);
124     }
125   }
126 
127   public boolean contains(StoragePath path) throws AssetStoreException {
128     try {
129       AwsAssetMapping map = database.findMapping(path);
130       return (map != null);
131     } catch (AwsAssetDatabaseException e) {
132       throw new AssetStoreException(e);
133     }
134   }
135 
136   protected File getFileFromWorkspace(Source source) {
137     try {
138       return workspace.get(source.getUri());
139     } catch (NotFoundException e) {
140       logger.error("Source file '{}' does not exist", source.getUri());
141       throw new AssetStoreException(e);
142     } catch (IOException e) {
143       logger.error("Error while getting file '{}' from workspace: {}", source.getUri(), getMessage(e));
144       throw new AssetStoreException(e);
145     }
146   }
147 
148   public String buildObjectName(File origin, StoragePath storagePath) {
149     // origin file name from workspace will be called
150     // WORKSPACE/http_ADMIN_HOST/assets/assets/MP_ID/EL_ID/VERSION/filename.EXTENSION
151     // while the actual file is in ARCHIVE/ORG/MP_ID/VERSION/EL_ID.EXTENSION
152 
153     // Create object key - S3 style but works for Glacier as well
154     String fileExt = FilenameUtils.getExtension(origin.getName());
155     return buildFilename(storagePath, fileExt.isEmpty() ? "" : "." + fileExt);
156   }
157 
158   /**
159    * Builds the aws object name.
160    */
161   protected String buildFilename(StoragePath path, String ext) {
162     // Something like ORG_ID/MP_ID/VERSION/ELEMENT_ID.EXTENSION
163     return StringUtils.join(new String[] { path.getOrganizationId(), path.getMediaPackageId(),
164             path.getVersion().toString(), path.getMediaPackageElementId() + ext }, "/");
165   }
166 
167   /**
168    * @see AssetStore#put(StoragePath, Source)
169    */
170   public void put(StoragePath storagePath, Source source) throws AssetStoreException {
171     // If the workspace  to asset manager hard-linking is enabled then this is just a
172     // hard-link. If not, this will be a download + hard-link
173     final File origin = getFileFromWorkspace(source);
174 
175     String objectName = buildObjectName(origin, storagePath);
176     String objectVersion = null;
177     try {
178       // Upload file to AWS
179       AwsUploadOperationResult result = uploadObject(origin, objectName, source.getMimeType());
180       objectName = result.getObjectName();
181       objectVersion = result.getObjectVersion();
182     } catch (Exception e) {
183       throw new AssetStoreException(e);
184     }
185 
186     try {
187       // Upload was successful. Store mapping in the database
188       logger.debug("Adding AWS {} mapping to database: {} points to {}, object version {}", getStoreType(),
189               storagePath, objectName, objectVersion);
190       database.storeMapping(storagePath, objectName, objectVersion);
191     } catch (AwsAssetDatabaseException e) {
192       throw new AssetStoreException(e);
193     }
194   }
195 
196   protected abstract AwsUploadOperationResult uploadObject(File origin, String objectName, Optional<MimeType> mimeType)
197           throws AssetStoreException;
198 
199   /** @see AssetStore#get(StoragePath) */
200   public Optional<InputStream> get(final StoragePath path) throws AssetStoreException {
201     try {
202       AwsAssetMapping map = database.findMapping(path);
203       if (map == null) {
204         logger.warn("File mapping not found in database: {}", path);
205         return Optional.empty();
206       }
207 
208       logger.debug("Getting archive object from AWS {}: {}", getStoreType(), map.getObjectKey());
209       return Optional.of(getObject(map));
210 
211     } catch (AssetStoreException e) {
212       throw e;
213     } catch (AwsAssetDatabaseException e) {
214       throw new AssetStoreException(e);
215     }
216   }
217 
218   protected abstract InputStream getObject(AwsAssetMapping map) throws AssetStoreException;
219 
220   /** @see AssetStore#delete(DeletionSelector) */
221   public boolean delete(DeletionSelector sel) throws AssetStoreException {
222     // Build path, version may be null if all versions are desired
223     StoragePath path = new StoragePath(
224         sel.getOrganizationId(), sel.getMediaPackageId(), sel.getVersion().orElse(null), null);
225     try {
226       List<AwsAssetMapping> list = database.findMappingsByMediaPackageAndVersion(path);
227       // Traverse all file mappings for that media package / version(s)
228       for (AwsAssetMapping map : list) {
229         // Find all mappings that point to the same object (like hard-links)
230         List<AwsAssetMapping> links = database.findMappingsByKey(map.getObjectKey());
231         if (links.size() == 1) {
232           // This is the only active mapping thats point to the object; thus, the object can be deleted.
233           logger.debug("Deleting archive object from AWS {}: {}, version {}",
234               getStoreType(), map.getObjectKey(), map.getObjectVersion());
235           deleteObject(map);
236           logger.info("Archive object deleted from AWS {}: {}, version {}",
237               getStoreType(), map.getObjectKey(), map.getObjectVersion());
238         }
239         // Add a deletion date to the mapping in the table. This doesn't delete the row.
240         database.deleteMapping(new StoragePath(
241             map.getOrganizationId(),
242             map.getMediaPackageId(),
243             new VersionImpl(map.getVersion()),
244             map.getMediaPackageElementId()));
245       }
246       return true;
247     } catch (AwsAssetDatabaseException e) {
248       throw new AssetStoreException(e);
249     }
250   }
251 
252   protected abstract void deleteObject(AwsAssetMapping map) throws AssetStoreException;
253 }