1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.assetmanager.aws.s3.endpoint;
23
24 import static org.opencastproject.util.RestUtil.R.noContent;
25 import static org.opencastproject.util.RestUtil.R.notFound;
26 import static org.opencastproject.util.RestUtil.R.ok;
27
28 import org.opencastproject.assetmanager.api.AssetManager;
29 import org.opencastproject.assetmanager.api.AssetManagerException;
30 import org.opencastproject.assetmanager.api.Snapshot;
31 import org.opencastproject.assetmanager.api.storage.AssetStoreException;
32 import org.opencastproject.assetmanager.api.storage.StoragePath;
33 import org.opencastproject.assetmanager.aws.s3.AwsS3AssetStore;
34 import org.opencastproject.mediapackage.MediaPackageElement;
35 import org.opencastproject.security.api.SecurityService;
36 import org.opencastproject.util.NotFoundException;
37 import org.opencastproject.util.doc.rest.RestParameter;
38 import org.opencastproject.util.doc.rest.RestQuery;
39 import org.opencastproject.util.doc.rest.RestResponse;
40 import org.opencastproject.util.doc.rest.RestService;
41
42 import com.amazonaws.services.s3.model.StorageClass;
43
44 import org.apache.commons.lang3.StringUtils;
45 import org.osgi.service.component.annotations.Component;
46 import org.osgi.service.component.annotations.Reference;
47 import org.osgi.service.jaxrs.whiteboard.propertytypes.JaxrsResource;
48 import org.slf4j.Logger;
49 import org.slf4j.LoggerFactory;
50
51 import java.util.Optional;
52 import java.util.function.Supplier;
53
54 import javax.servlet.http.HttpServletResponse;
55 import javax.ws.rs.BadRequestException;
56 import javax.ws.rs.FormParam;
57 import javax.ws.rs.GET;
58 import javax.ws.rs.PUT;
59 import javax.ws.rs.Path;
60 import javax.ws.rs.PathParam;
61 import javax.ws.rs.Produces;
62 import javax.ws.rs.WebApplicationException;
63 import javax.ws.rs.core.MediaType;
64 import javax.ws.rs.core.Response;
65
66 @Path("/assets/aws/s3")
67 @RestService(name = "archive-aws-s3", title = "AWS S3 Archive",
68 notes = {
69 "All paths are relative to the REST endpoint base (something like http://your.server/files)",
70 "If you notice that this service is not working as expected, there might be a bug! "
71 + "You should file an error report with your server logs from the time when the error occurred: "
72 + "<a href=\"http://opencast.jira.com\">Opencast Issue Tracker</a>"
73 },
74 abstractText = "This service handles AWS S3 archived assets")
75 @Component(
76 immediate = true,
77 service = AwsS3RestEndpoint.class,
78 property = {
79 "service.description=AssetManager S3 REST Endpoint",
80 "opencast.service.type=org.opencastproject.assetmanager.aws-s3",
81 "opencast.service.path=/assets/aws/s3",
82 }
83 )
84 @JaxrsResource
85 public class AwsS3RestEndpoint {
86
87 private static final Logger logger = LoggerFactory.getLogger(AwsS3RestEndpoint.class);
88
89 private AwsS3AssetStore awsS3AssetStore = null;
90 private AssetManager assetManager = null;
91 private SecurityService securityService = null;
92
93 @GET
94 @Path("{mediaPackageId}/assets/storageClass")
95 @Produces(MediaType.TEXT_PLAIN)
96 @RestQuery(name = "getStorageClass",
97 description = "Get the S3 Storage Class for each asset in the Media Package",
98 pathParameters = {
99 @RestParameter(
100 name = "mediaPackageId", isRequired = true,
101 type = RestParameter.Type.STRING,
102 description = "The media package indentifier.")},
103 responses = {
104 @RestResponse(
105 description = "mediapackage found in S3",
106 responseCode = HttpServletResponse.SC_OK),
107 @RestResponse(
108 description = "mediapackage not found or has no assets in S3",
109 responseCode = HttpServletResponse.SC_NOT_FOUND)
110 },
111 returnDescription = "List each assets's Object Key and S3 Storage Class")
112 public Response getStorageClass(@PathParam("mediaPackageId") final String mediaPackageId) {
113 return handleException(() -> {
114 String mpId = StringUtils.trimToNull(mediaPackageId);
115
116 Optional<Snapshot> snapshot = assetManager.getLatestSnapshot(mpId);
117 if (snapshot.isEmpty()) {
118 return notFound();
119 }
120
121 StringBuilder info = new StringBuilder();
122 for (MediaPackageElement e : snapshot.get().getMediaPackage().elements()) {
123 if (e.getElementType() == MediaPackageElement.Type.Publication) {
124 continue;
125 }
126
127 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
128 mpId,
129 snapshot.get().getVersion(),
130 e.getIdentifier());
131 if (awsS3AssetStore.contains(storagePath)) {
132 try {
133 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
134 awsS3AssetStore.getAssetStorageClass(storagePath)));
135 } catch (AssetStoreException ex) {
136 throw new AssetManagerException(ex);
137 }
138 } else {
139 info.append(String.format("%s,NONE\n", e.getURI()));
140 }
141 }
142 return ok(info.toString());
143 });
144 }
145
146 @PUT
147 @Path("{mediaPackageId}/assets")
148 @Produces(MediaType.TEXT_PLAIN)
149 @RestQuery(name = "modifyStorageClass",
150 description = "Move the Media Package assets to the specified S3 Storage Class if possible",
151 pathParameters = {
152 @RestParameter(
153 name = "mediaPackageId",
154 isRequired = true,
155 type = RestParameter.Type.STRING,
156 description = "The media package indentifier.")
157 },
158 restParameters = {
159 @RestParameter(
160 name = "storageClass",
161 isRequired = true,
162 type = RestParameter.Type.STRING,
163 description = "The S3 storage class, valid terms STANDARD, STANDARD_IA, INTELLIGENT_TIERING, ONEZONE_IA,"
164 + "GLACIER_IR, GLACIER, and DEEP_ARCHIVE. See https://aws.amazon.com/s3/storage-classes/")
165 },
166 responses = {
167 @RestResponse(
168 description = "mediapackage found in S3",
169 responseCode = HttpServletResponse.SC_OK),
170 @RestResponse(
171 description = "mediapackage not found or has no assets in S3",
172 responseCode = HttpServletResponse.SC_NOT_FOUND) },
173 returnDescription = "List each asset's Object Key and new S3 Storage Class")
174 public Response modifyStorageClass(@PathParam("mediaPackageId") final String mediaPackageId,
175 @FormParam("storageClass") final String storageClass) {
176 return handleException(() -> {
177 String mpId = StringUtils.trimToNull(mediaPackageId);
178 String sc = StringUtils.trimToNull(storageClass);
179
180 Optional<Snapshot> snapshot = assetManager.getLatestSnapshot(mpId);
181 if (snapshot.isEmpty()) {
182 return notFound();
183 }
184 StringBuilder info = new StringBuilder();
185 for (MediaPackageElement e : snapshot.get().getMediaPackage().elements()) {
186 if (e.getElementType() == MediaPackageElement.Type.Publication) {
187 continue;
188 }
189
190 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
191 mpId,
192 snapshot.get().getVersion(),
193 e.getIdentifier());
194 if (awsS3AssetStore.contains(storagePath)) {
195 try {
196 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
197 awsS3AssetStore.modifyAssetStorageClass(storagePath, sc)));
198 } catch (AssetStoreException ex) {
199 throw new AssetManagerException(ex);
200 }
201 } else {
202 info.append(String.format("%s,NONE\n", e.getURI()));
203 }
204 }
205 return ok(info.toString());
206 });
207 }
208
209 @GET
210 @Path("glacier/{mediaPackageId}/assets")
211 @Produces(MediaType.TEXT_PLAIN)
212 @RestQuery(name = "restoreAssetsStatus",
213 description = "Get the mediapackage asset's restored status",
214 pathParameters = {
215 @RestParameter(
216 name = "mediaPackageId",
217 isRequired = true,
218 type = RestParameter.Type.STRING,
219 description = "The media package indentifier.")
220 },
221 responses = {
222 @RestResponse(
223 description = "mediapackage found in S3 and assets in Glacier",
224 responseCode = HttpServletResponse.SC_OK),
225 @RestResponse(
226 description = "mediapackage found in S3 but no assets in Glacier",
227 responseCode = HttpServletResponse.SC_NO_CONTENT),
228 @RestResponse(
229 description = "mediapackage not found or has no assets in S3",
230 responseCode = HttpServletResponse.SC_NOT_FOUND)
231 },
232 returnDescription = "List each glacier asset's restoration status and expiration date")
233 public Response getAssetRestoreState(@PathParam("mediaPackageId") final String mediaPackageId) {
234 return handleException(() -> {
235 String mpId = StringUtils.trimToNull(mediaPackageId);
236
237 Optional<Snapshot> snapshot = assetManager.getLatestSnapshot(mpId);
238 if (snapshot.isEmpty()) {
239 return notFound();
240 }
241
242 StringBuilder info = new StringBuilder();
243 for (MediaPackageElement e : snapshot.get().getMediaPackage().elements()) {
244 if (e.getElementType() == MediaPackageElement.Type.Publication) {
245 continue;
246 }
247
248 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
249 mpId,
250 snapshot.get().getVersion(),
251 e.getIdentifier());
252 if (isFrozen(storagePath)) {
253 try {
254 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
255 awsS3AssetStore.getAssetRestoreStatusString(storagePath)));
256 } catch (AssetStoreException ex) {
257 throw new AssetManagerException(ex);
258 }
259 } else {
260 info.append(String.format("%s,NONE\n", storagePath));
261 }
262 }
263 if (info.length() == 0) {
264 return noContent();
265 }
266 return ok(info.toString());
267 });
268 }
269
270 @PUT
271 @Path("glacier/{mediaPackageId}/assets")
272 @Produces(MediaType.TEXT_PLAIN)
273 @RestQuery(name = "restoreAssets",
274 description = "Initiate the restore of any assets in Glacier storage class",
275 pathParameters = {
276 @RestParameter(
277 name = "mediaPackageId",
278 isRequired = true,
279 type = RestParameter.Type.STRING,
280 description = "The media package indentifier.")
281 },
282 restParameters = {
283 @RestParameter(
284 name = "restorePeriod",
285 isRequired = false,
286 type = RestParameter.Type.INTEGER,
287 defaultValue = "2",
288 description = "Number of days to restore the assets for, default see service configuration")
289 },
290 responses = {
291 @RestResponse(
292 description = "restore of assets started",
293 responseCode = HttpServletResponse.SC_NO_CONTENT),
294 @RestResponse(
295 description = "invalid restore period, must be greater than zero",
296 responseCode = HttpServletResponse.SC_BAD_REQUEST),
297 @RestResponse(
298 description = "mediapackage not found or has no assets in S3",
299 responseCode = HttpServletResponse.SC_NOT_FOUND)
300 },
301 returnDescription = "Restore of assets initiated")
302 public Response restoreAssets(@PathParam("mediaPackageId") final String mediaPackageId,
303 @FormParam("restorePeriod") final Integer restorePeriod) {
304 return handleException(() -> {
305 String mpId = StringUtils.trimToNull(mediaPackageId);
306 Integer rp = restorePeriod != null ? restorePeriod : awsS3AssetStore.getRestorePeriod();
307
308 if (rp < 1) {
309 throw new BadRequestException("Restore period must be greater than zero!");
310 }
311
312 Optional<Snapshot> snapshot = assetManager.getLatestSnapshot(mpId);
313 if (snapshot.isEmpty()) {
314 return notFound();
315 }
316
317 for (MediaPackageElement e : snapshot.get().getMediaPackage().elements()) {
318 if (e.getElementType() == MediaPackageElement.Type.Publication) {
319 continue;
320 }
321
322 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
323 mpId,
324 snapshot.get().getVersion(),
325 e.getIdentifier());
326 if (isFrozen(storagePath)) {
327 try {
328
329 awsS3AssetStore.initiateRestoreAsset(storagePath, rp);
330 } catch (AssetStoreException ex) {
331 throw new AssetManagerException(ex);
332 }
333 }
334 }
335 return noContent();
336 });
337 }
338
339 private boolean isFrozen(StoragePath storagePath) {
340 String assetStorageClass = awsS3AssetStore.getAssetStorageClass(storagePath);
341 return awsS3AssetStore.contains(storagePath)
342 && (StorageClass.Glacier == StorageClass.fromValue(assetStorageClass)
343 || StorageClass.DeepArchive == StorageClass.fromValue(assetStorageClass));
344 }
345
346
347
348 public static <A> A handleException(Supplier<A> f) {
349 try {
350 return f.get();
351 } catch (AssetManagerException e) {
352 if (e.isCauseNotAuthorized()) {
353 throw new WebApplicationException(e, Response.Status.UNAUTHORIZED);
354 }
355 if (e.isCauseNotFound()) {
356 throw new WebApplicationException(e, Response.Status.NOT_FOUND);
357 }
358 throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
359 } catch (Exception e) {
360 logger.error("Error calling archive REST method", e);
361 if (e instanceof NotFoundException) {
362 throw new WebApplicationException(e, Response.Status.NOT_FOUND);
363 }
364 throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
365 }
366 }
367
368 @Reference()
369 void setAwsS3AssetStore(AwsS3AssetStore store) {
370 awsS3AssetStore = store;
371 }
372
373 @Reference()
374 void setAssetManager(AssetManager service) {
375 assetManager = service;
376 }
377
378 @Reference()
379 void setSecurityService(SecurityService service) {
380 securityService = service;
381 }
382 }