1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.assetmanager.aws.s3.endpoint;
23
24 import static org.opencastproject.util.RestUtil.R.noContent;
25 import static org.opencastproject.util.RestUtil.R.notFound;
26 import static org.opencastproject.util.RestUtil.R.ok;
27 import static org.opencastproject.util.RestUtil.R.serverError;
28
29 import org.opencastproject.assetmanager.api.AssetManager;
30 import org.opencastproject.assetmanager.api.AssetManagerException;
31 import org.opencastproject.assetmanager.api.query.AQueryBuilder;
32 import org.opencastproject.assetmanager.api.query.ARecord;
33 import org.opencastproject.assetmanager.api.query.AResult;
34 import org.opencastproject.assetmanager.api.query.ASelectQuery;
35 import org.opencastproject.assetmanager.api.storage.AssetStoreException;
36 import org.opencastproject.assetmanager.api.storage.StoragePath;
37 import org.opencastproject.assetmanager.aws.s3.AwsS3AssetStore;
38 import org.opencastproject.mediapackage.MediaPackageElement;
39 import org.opencastproject.security.api.SecurityService;
40 import org.opencastproject.util.NotFoundException;
41 import org.opencastproject.util.data.Function0;
42 import org.opencastproject.util.doc.rest.RestParameter;
43 import org.opencastproject.util.doc.rest.RestQuery;
44 import org.opencastproject.util.doc.rest.RestResponse;
45 import org.opencastproject.util.doc.rest.RestService;
46
47 import com.amazonaws.services.s3.model.StorageClass;
48
49 import org.apache.commons.lang3.StringUtils;
50 import org.osgi.service.component.annotations.Component;
51 import org.osgi.service.component.annotations.Reference;
52 import org.osgi.service.jaxrs.whiteboard.propertytypes.JaxrsResource;
53 import org.slf4j.Logger;
54 import org.slf4j.LoggerFactory;
55
56 import javax.servlet.http.HttpServletResponse;
57 import javax.ws.rs.BadRequestException;
58 import javax.ws.rs.FormParam;
59 import javax.ws.rs.GET;
60 import javax.ws.rs.PUT;
61 import javax.ws.rs.Path;
62 import javax.ws.rs.PathParam;
63 import javax.ws.rs.Produces;
64 import javax.ws.rs.WebApplicationException;
65 import javax.ws.rs.core.MediaType;
66 import javax.ws.rs.core.Response;
67
68 @Path("/assets/aws/s3")
69 @RestService(name = "archive-aws-s3", title = "AWS S3 Archive",
70 notes = {
71 "All paths are relative to the REST endpoint base (something like http://your.server/files)",
72 "If you notice that this service is not working as expected, there might be a bug! "
73 + "You should file an error report with your server logs from the time when the error occurred: "
74 + "<a href=\"http://opencast.jira.com\">Opencast Issue Tracker</a>"
75 },
76 abstractText = "This service handles AWS S3 archived assets")
77 @Component(
78 immediate = true,
79 service = AwsS3RestEndpoint.class,
80 property = {
81 "service.description=AssetManager S3 REST Endpoint",
82 "opencast.service.type=org.opencastproject.assetmanager.aws-s3",
83 "opencast.service.path=/assets/aws/s3",
84 }
85 )
86 @JaxrsResource
87 public class AwsS3RestEndpoint {
88
89 private static final Logger logger = LoggerFactory.getLogger(AwsS3RestEndpoint.class);
90
91 private AwsS3AssetStore awsS3AssetStore = null;
92 private AssetManager assetManager = null;
93 private SecurityService securityService = null;
94
95 @GET
96 @Path("{mediaPackageId}/assets/storageClass")
97 @Produces(MediaType.TEXT_PLAIN)
98 @RestQuery(name = "getStorageClass",
99 description = "Get the S3 Storage Class for each asset in the Media Package",
100 pathParameters = {
101 @RestParameter(
102 name = "mediaPackageId", isRequired = true,
103 type = RestParameter.Type.STRING,
104 description = "The media package indentifier.")},
105 responses = {
106 @RestResponse(
107 description = "mediapackage found in S3",
108 responseCode = HttpServletResponse.SC_OK),
109 @RestResponse(
110 description = "mediapackage not found or has no assets in S3",
111 responseCode = HttpServletResponse.SC_NOT_FOUND)
112 },
113 returnDescription = "List each assets's Object Key and S3 Storage Class")
114 public Response getStorageClass(@PathParam("mediaPackageId") final String mediaPackageId) {
115 return handleException(new Function0<Response>() {
116 private String getMediaPackageId() {
117 return StringUtils.trimToNull(mediaPackageId);
118 }
119
120 @Override public Response apply() {
121 AQueryBuilder q = assetManager.createQuery();
122 final ASelectQuery idQuery = q.select(q.snapshot())
123 .where(
124 q.organizationId(securityService.getOrganization().getId())
125 .and(q.mediaPackageId(getMediaPackageId()))
126 .and(q.version().isLatest()));
127 final AResult result = idQuery.run();
128 if (result.getSize() > 1) {
129 return serverError();
130 }
131 if (result.getSize() == 0) {
132 return notFound();
133 }
134 final ARecord item = result.getRecords().stream().findFirst().get();
135
136 StringBuilder info = new StringBuilder();
137 for (MediaPackageElement e : assetManager.getMediaPackage(item.getMediaPackageId()).get().elements()) {
138 if (e.getElementType() == MediaPackageElement.Type.Publication) {
139 continue;
140 }
141
142 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
143 getMediaPackageId(),
144 item.getSnapshot().get().getVersion(),
145 e.getIdentifier());
146 if (awsS3AssetStore.contains(storagePath)) {
147 try {
148 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
149 awsS3AssetStore.getAssetStorageClass(storagePath)));
150 } catch (AssetStoreException ex) {
151 throw new AssetManagerException(ex);
152 }
153 } else {
154 info.append(String.format("%s,NONE\n", e.getURI()));
155 }
156 }
157 return ok(info.toString());
158 }
159
160 });
161 }
162
163 @PUT
164 @Path("{mediaPackageId}/assets")
165 @Produces(MediaType.TEXT_PLAIN)
166 @RestQuery(name = "modifyStorageClass",
167 description = "Move the Media Package assets to the specified S3 Storage Class if possible",
168 pathParameters = {
169 @RestParameter(
170 name = "mediaPackageId",
171 isRequired = true,
172 type = RestParameter.Type.STRING,
173 description = "The media package indentifier.")
174 },
175 restParameters = {
176 @RestParameter(
177 name = "storageClass",
178 isRequired = true,
179 type = RestParameter.Type.STRING,
180 description = "The S3 storage class, valid terms STANDARD, STANDARD_IA, INTELLIGENT_TIERING, ONEZONE_IA,"
181 + "GLACIER_IR, GLACIER, and DEEP_ARCHIVE. See https://aws.amazon.com/s3/storage-classes/")
182 },
183 responses = {
184 @RestResponse(
185 description = "mediapackage found in S3",
186 responseCode = HttpServletResponse.SC_OK),
187 @RestResponse(
188 description = "mediapackage not found or has no assets in S3",
189 responseCode = HttpServletResponse.SC_NOT_FOUND) },
190 returnDescription = "List each asset's Object Key and new S3 Storage Class")
191 public Response modifyStorageClass(@PathParam("mediaPackageId") final String mediaPackageId,
192 @FormParam("storageClass") final String storageClass) {
193 return handleException(new Function0<Response>() {
194 private String getMediaPackageId() {
195 return StringUtils.trimToNull(mediaPackageId);
196 }
197
198 private String getStorageClass() {
199 return StringUtils.trimToNull(storageClass);
200 }
201
202 @Override public Response apply() {
203 AQueryBuilder q = assetManager.createQuery();
204 final ASelectQuery idQuery = q.select(q.snapshot())
205 .where(
206 q.organizationId(securityService.getOrganization().getId())
207 .and(q.mediaPackageId(getMediaPackageId()))
208 .and(q.version().isLatest()));
209 final AResult result = idQuery.run();
210 if (result.getSize() > 1) {
211 return serverError();
212 }
213 if (result.getSize() == 0) {
214 return notFound();
215 }
216 final ARecord item = result.getRecords().stream().findFirst().get();
217
218 StringBuilder info = new StringBuilder();
219 for (MediaPackageElement e : assetManager.getMediaPackage(item.getMediaPackageId()).get().elements()) {
220 if (e.getElementType() == MediaPackageElement.Type.Publication) {
221 continue;
222 }
223
224 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
225 getMediaPackageId(),
226 item.getSnapshot().get().getVersion(),
227 e.getIdentifier());
228 if (awsS3AssetStore.contains(storagePath)) {
229 try {
230 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
231 awsS3AssetStore.modifyAssetStorageClass(storagePath,
232 getStorageClass())));
233 } catch (AssetStoreException ex) {
234 throw new AssetManagerException(ex);
235 }
236 } else {
237 info.append(String.format("%s,NONE\n", e.getURI()));
238 }
239 }
240 return ok(info.toString());
241 }
242
243 });
244 }
245
246 @GET
247 @Path("glacier/{mediaPackageId}/assets")
248 @Produces(MediaType.TEXT_PLAIN)
249 @RestQuery(name = "restoreAssetsStatus",
250 description = "Get the mediapackage asset's restored status",
251 pathParameters = {
252 @RestParameter(
253 name = "mediaPackageId",
254 isRequired = true,
255 type = RestParameter.Type.STRING,
256 description = "The media package indentifier.")
257 },
258 responses = {
259 @RestResponse(
260 description = "mediapackage found in S3 and assets in Glacier",
261 responseCode = HttpServletResponse.SC_OK),
262 @RestResponse(
263 description = "mediapackage found in S3 but no assets in Glacier",
264 responseCode = HttpServletResponse.SC_NO_CONTENT),
265 @RestResponse(
266 description = "mediapackage not found or has no assets in S3",
267 responseCode = HttpServletResponse.SC_NOT_FOUND)
268 },
269 returnDescription = "List each glacier asset's restoration status and expiration date")
270 public Response getAssetRestoreState(@PathParam("mediaPackageId") final String mediaPackageId) {
271 return handleException(new Function0<Response>() {
272 private String getMediaPackageId() {
273 return StringUtils.trimToNull(mediaPackageId);
274 }
275
276 @Override public Response apply() {
277 AQueryBuilder q = assetManager.createQuery();
278 final ASelectQuery idQuery = q.select(q.snapshot())
279 .where(
280 q.organizationId(securityService.getOrganization().getId())
281 .and(q.mediaPackageId(getMediaPackageId()))
282 .and(q.version().isLatest()));
283 final AResult result = idQuery.run();
284 if (result.getSize() > 1) {
285 return serverError();
286 }
287 if (result.getSize() == 0) {
288 return notFound();
289 }
290 final ARecord item = result.getRecords().stream().findFirst().get();
291
292 StringBuilder info = new StringBuilder();
293 for (MediaPackageElement e : assetManager.getMediaPackage(item.getMediaPackageId()).get().elements()) {
294 if (e.getElementType() == MediaPackageElement.Type.Publication) {
295 continue;
296 }
297
298 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
299 getMediaPackageId(),
300 item.getSnapshot().get().getVersion(),
301 e.getIdentifier());
302 if (isFrozen(storagePath)) {
303 try {
304 info.append(String.format("%s,%s\n", awsS3AssetStore.getAssetObjectKey(storagePath),
305 awsS3AssetStore.getAssetRestoreStatusString(storagePath)));
306 } catch (AssetStoreException ex) {
307 throw new AssetManagerException(ex);
308 }
309 } else {
310 info.append(String.format("%s,NONE\n", storagePath));
311 }
312 }
313 if (info.length() == 0) {
314 return noContent();
315 }
316 return ok(info.toString());
317 }
318 });
319 }
320
321 @PUT
322 @Path("glacier/{mediaPackageId}/assets")
323 @Produces(MediaType.TEXT_PLAIN)
324 @RestQuery(name = "restoreAssets",
325 description = "Initiate the restore of any assets in Glacier storage class",
326 pathParameters = {
327 @RestParameter(
328 name = "mediaPackageId",
329 isRequired = true,
330 type = RestParameter.Type.STRING,
331 description = "The media package indentifier.")
332 },
333 restParameters = {
334 @RestParameter(
335 name = "restorePeriod",
336 isRequired = false,
337 type = RestParameter.Type.INTEGER,
338 defaultValue = "2",
339 description = "Number of days to restore the assets for, default see service configuration")
340 },
341 responses = {
342 @RestResponse(
343 description = "restore of assets started",
344 responseCode = HttpServletResponse.SC_NO_CONTENT),
345 @RestResponse(
346 description = "invalid restore period, must be greater than zero",
347 responseCode = HttpServletResponse.SC_BAD_REQUEST),
348 @RestResponse(
349 description = "mediapackage not found or has no assets in S3",
350 responseCode = HttpServletResponse.SC_NOT_FOUND)
351 },
352 returnDescription = "Restore of assets initiated")
353 public Response restoreAssets(@PathParam("mediaPackageId") final String mediaPackageId,
354 @FormParam("restorePeriod") final Integer restorePeriod) {
355 return handleException(new Function0<Response>() {
356 private String getMediaPackageId() {
357 return StringUtils.trimToNull(mediaPackageId);
358 }
359
360 private Integer getRestorePeriod() {
361 return restorePeriod != null ? restorePeriod : awsS3AssetStore.getRestorePeriod();
362 }
363
364 @Override public Response apply() {
365 Integer restorePeriod = getRestorePeriod();
366 if (restorePeriod < 1) {
367 throw new BadRequestException("Restore period must be greater than zero!");
368 }
369
370 AQueryBuilder q = assetManager.createQuery();
371 final ASelectQuery idQuery = q.select(q.snapshot())
372 .where(
373 q.organizationId(securityService.getOrganization().getId())
374 .and(q.mediaPackageId(getMediaPackageId()))
375 .and(q.version().isLatest()));
376 final AResult result = idQuery.run();
377 if (result.getSize() > 1) {
378 return serverError();
379 }
380 if (result.getSize() == 0) {
381 return notFound();
382 }
383 final ARecord item = result.getRecords().stream().findFirst().get();
384
385
386 for (MediaPackageElement e : assetManager.getMediaPackage(item.getMediaPackageId()).get().elements()) {
387 if (e.getElementType() == MediaPackageElement.Type.Publication) {
388 continue;
389 }
390
391 StoragePath storagePath = new StoragePath(securityService.getOrganization().getId(),
392 getMediaPackageId(),
393 item.getSnapshot().get().getVersion(),
394 e.getIdentifier());
395 if (isFrozen(storagePath)) {
396 try {
397
398 awsS3AssetStore.initiateRestoreAsset(storagePath, getRestorePeriod());
399 } catch (AssetStoreException ex) {
400 throw new AssetManagerException(ex);
401 }
402 }
403 }
404 return noContent();
405 }
406 });
407 }
408
409 private boolean isFrozen(StoragePath storagePath) {
410 String assetStorageClass = awsS3AssetStore.getAssetStorageClass(storagePath);
411 return awsS3AssetStore.contains(storagePath)
412 && (StorageClass.Glacier == StorageClass.fromValue(assetStorageClass)
413 || StorageClass.DeepArchive == StorageClass.fromValue(assetStorageClass));
414 }
415
416
417
418 public static <A> A handleException(final Function0<A> f) {
419 try {
420 return f.apply();
421 } catch (AssetManagerException e) {
422 if (e.isCauseNotAuthorized()) {
423 throw new WebApplicationException(e, Response.Status.UNAUTHORIZED);
424 }
425 if (e.isCauseNotFound()) {
426 throw new WebApplicationException(e, Response.Status.NOT_FOUND);
427 }
428 throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
429 } catch (Exception e) {
430 logger.error("Error calling archive REST method", e);
431 if (e instanceof NotFoundException) {
432 throw new WebApplicationException(e, Response.Status.NOT_FOUND);
433 }
434 throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
435 }
436 }
437
438 @Reference()
439 void setAwsS3AssetStore(AwsS3AssetStore store) {
440 awsS3AssetStore = store;
441 }
442
443 @Reference()
444 void setAssetManager(AssetManager service) {
445 assetManager = service;
446 }
447
448 @Reference()
449 void setSecurityService(SecurityService service) {
450 securityService = service;
451 }
452 }