View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  
22  package org.opencastproject.assetmanager.aws;
23  
24  import static org.apache.commons.lang3.exception.ExceptionUtils.getMessage;
25  
26  import org.opencastproject.assetmanager.api.storage.AssetStore;
27  import org.opencastproject.assetmanager.api.storage.AssetStoreException;
28  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
29  import org.opencastproject.assetmanager.api.storage.Source;
30  import org.opencastproject.assetmanager.api.storage.StoragePath;
31  import org.opencastproject.assetmanager.aws.persistence.AwsAssetDatabase;
32  import org.opencastproject.assetmanager.aws.persistence.AwsAssetDatabaseException;
33  import org.opencastproject.assetmanager.aws.persistence.AwsAssetMapping;
34  import org.opencastproject.assetmanager.impl.VersionImpl;
35  import org.opencastproject.util.ConfigurationException;
36  import org.opencastproject.util.MimeType;
37  import org.opencastproject.util.NotFoundException;
38  import org.opencastproject.util.OsgiUtil;
39  import org.opencastproject.util.data.Option;
40  import org.opencastproject.workspace.api.Workspace;
41  
42  import org.apache.commons.io.FilenameUtils;
43  import org.apache.commons.lang3.StringUtils;
44  import org.osgi.service.component.ComponentContext;
45  import org.slf4j.Logger;
46  import org.slf4j.LoggerFactory;
47  
48  import java.io.File;
49  import java.io.IOException;
50  import java.io.InputStream;
51  import java.util.List;
52  import java.util.Optional;
53  
54  public abstract class AwsAbstractArchive implements AssetStore {
55  
56    /** Log facility */
57    private static final Logger logger = LoggerFactory.getLogger(AwsAbstractArchive.class);
58  
59    protected Workspace workspace;
60    protected AwsAssetDatabase database;
61  
62    /** The store type e.g. aws (long-term), or other implementations */
63    protected String storeType = null;
64    /** The AWS region */
65    protected String regionName = null;
66  
67    protected String getAWSConfigKey(ComponentContext cc, String key) {
68      try {
69        String value = StringUtils.trimToEmpty(OsgiUtil.getComponentContextProperty(cc, key));
70        if (StringUtils.isNotBlank(value)) {
71          return value;
72        }
73        throw new ConfigurationException(key + " is invalid");
74      } catch (RuntimeException e) {
75        throw new ConfigurationException(key + " is missing or invalid", e);
76      }
77    }
78  
79    public Option<Long> getUsedSpace() {
80      throw new UnsupportedOperationException("Not implemented");
81    }
82  
83    public Option<Long> getUsableSpace() {
84      throw new UnsupportedOperationException("Not implemented");
85    }
86  
87    public Option<Long> getTotalSpace() {
88      throw new UnsupportedOperationException("Not implemented");
89    }
90  
91    public String getStoreType() {
92      return this.storeType;
93    }
94  
95    public String getRegion() {
96      return this.regionName;
97    }
98  
99    /** OSGi Di */
100   public void setWorkspace(Workspace workspace) {
101     this.workspace = workspace;
102   }
103 
104   /** OSGi Di */
105   public void setDatabase(AwsAssetDatabase db) {
106     this.database = db;
107   }
108 
109   /** @see AssetStore#copy(StoragePath, StoragePath) */
110   public boolean copy(final StoragePath from, final StoragePath to) throws AssetStoreException {
111     try {
112       AwsAssetMapping map = database.findMapping(from);
113       if (!contains(from)) {
114         logger.warn("Origin file mapping not found in database: {}", from);
115         return false;
116       }
117       // New mapping will point to the SAME AWS object, nothing will be uploaded
118       logger.debug("Adding AWS {} link mapping to database: {} points to {}, version {}", getStoreType(),
119               to, map.getObjectKey(), map.getObjectVersion());
120       database.storeMapping(to, map.getObjectKey(), map.getObjectVersion());
121       return true;
122     } catch (AwsAssetDatabaseException e) {
123       throw new AssetStoreException(e);
124     }
125   }
126 
127   public boolean contains(StoragePath path) throws AssetStoreException {
128     try {
129       AwsAssetMapping map = database.findMapping(path);
130       return (map != null);
131     } catch (AwsAssetDatabaseException e) {
132       throw new AssetStoreException(e);
133     }
134   }
135 
136   protected File getFileFromWorkspace(Source source) {
137     try {
138       return workspace.get(source.getUri());
139     } catch (NotFoundException e) {
140       logger.error("Source file '{}' does not exist", source.getUri());
141       throw new AssetStoreException(e);
142     } catch (IOException e) {
143       logger.error("Error while getting file '{}' from workspace: {}", source.getUri(), getMessage(e));
144       throw new AssetStoreException(e);
145     }
146   }
147 
148   public String buildObjectName(File origin, StoragePath storagePath) {
149     // origin file name from workspace will be called
150     // WORKSPACE/http_ADMIN_HOST/assets/assets/MP_ID/EL_ID/VERSION/filename.EXTENSION
151     // while the actual file is in ARCHIVE/ORG/MP_ID/VERSION/EL_ID.EXTENSION
152 
153     // Create object key - S3 style but works for Glacier as well
154     String fileExt = FilenameUtils.getExtension(origin.getName());
155     return buildFilename(storagePath, fileExt.isEmpty() ? "" : "." + fileExt);
156   }
157 
158   /**
159    * Builds the aws object name.
160    */
161   protected String buildFilename(StoragePath path, String ext) {
162     // Something like ORG_ID/MP_ID/VERSION/ELEMENT_ID.EXTENSION
163     return StringUtils.join(new String[] { path.getOrganizationId(), path.getMediaPackageId(),
164             path.getVersion().toString(), path.getMediaPackageElementId() + ext }, "/");
165   }
166 
167   /**
168    * @see AssetStore#put(StoragePath, Source)
169    */
170   public void put(StoragePath storagePath, Source source) throws AssetStoreException {
171     // If the workspace  to asset manager hard-linking is enabled then this is just a
172     // hard-link. If not, this will be a download + hard-link
173     final File origin = getFileFromWorkspace(source);
174 
175     String objectName = buildObjectName(origin, storagePath);
176     String objectVersion = null;
177     try {
178       // Upload file to AWS
179       AwsUploadOperationResult result = uploadObject(storagePath.getOrganizationId(), origin, objectName,
180           source.getMimeType());
181       objectName = result.getObjectName();
182       objectVersion = result.getObjectVersion();
183     } catch (Exception e) {
184       throw new AssetStoreException(e);
185     }
186 
187     try {
188       // Upload was successful. Store mapping in the database
189       logger.debug("Adding AWS {} mapping to database: {} points to {}, object version {}", getStoreType(),
190               storagePath, objectName, objectVersion);
191       database.storeMapping(storagePath, objectName, objectVersion);
192     } catch (AwsAssetDatabaseException e) {
193       throw new AssetStoreException(e);
194     }
195   }
196 
197   protected abstract AwsUploadOperationResult uploadObject(String orgId, File origin, String objectName,
198           Optional<MimeType> mimeType) throws AssetStoreException;
199 
200   /** @see AssetStore#get(StoragePath) */
201   public Optional<InputStream> get(final StoragePath path) throws AssetStoreException {
202     try {
203       AwsAssetMapping map = database.findMapping(path);
204       if (map == null) {
205         logger.warn("File mapping not found in database: {}", path);
206         return Optional.empty();
207       }
208 
209       logger.debug("Getting archive object from AWS {}: {}", getStoreType(), map.getObjectKey());
210       return Optional.of(getObject(map));
211 
212     } catch (AssetStoreException e) {
213       throw e;
214     } catch (AwsAssetDatabaseException e) {
215       throw new AssetStoreException(e);
216     }
217   }
218 
219   protected abstract InputStream getObject(AwsAssetMapping map) throws AssetStoreException;
220 
221   /** @see AssetStore#delete(DeletionSelector) */
222   public boolean delete(DeletionSelector sel) throws AssetStoreException {
223     // Build path, version may be null if all versions are desired
224     StoragePath path = new StoragePath(
225         sel.getOrganizationId(), sel.getMediaPackageId(), sel.getVersion().orElse(null), null);
226     try {
227       List<AwsAssetMapping> list = database.findMappingsByMediaPackageAndVersion(path);
228       // Traverse all file mappings for that media package / version(s)
229       for (AwsAssetMapping map : list) {
230         // Find all mappings that point to the same object (like hard-links)
231         List<AwsAssetMapping> links = database.findMappingsByKey(map.getObjectKey());
232         if (links.size() == 1) {
233           // This is the only active mapping thats point to the object; thus, the object can be deleted.
234           logger.debug("Deleting archive object from AWS {}: {}, version {}",
235               getStoreType(), map.getObjectKey(), map.getObjectVersion());
236           deleteObject(map);
237           logger.info("Archive object deleted from AWS {}: {}, version {}",
238               getStoreType(), map.getObjectKey(), map.getObjectVersion());
239         }
240         // Add a deletion date to the mapping in the table. This doesn't delete the row.
241         database.deleteMapping(new StoragePath(
242             map.getOrganizationId(),
243             map.getMediaPackageId(),
244             new VersionImpl(map.getVersion()),
245             map.getMediaPackageElementId()));
246       }
247       return true;
248     } catch (AwsAssetDatabaseException e) {
249       throw new AssetStoreException(e);
250     }
251   }
252 
253   protected abstract void deleteObject(AwsAssetMapping map) throws AssetStoreException;
254 }