MicrosoftAzureStorageClient

/*
 * Licensed to The Apereo Foundation under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 *
 * The Apereo Foundation licenses this file to you under the Educational
 * Community License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License
 * at:
 *
 *   http://opensource.org/licenses/ecl2.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 */

package org.opencastproject.transcription.microsoft.azure;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;

public class MicrosoftAzureStorageClient {

  private static final Logger logger = LoggerFactory.getLogger(MicrosoftAzureStorageClient.class);

  private MicrosoftAzureAuthorization azureAuthorization;

  public MicrosoftAzureStorageClient(MicrosoftAzureAuthorization azureAuthorization) {
    this.azureAuthorization = azureAuthorization;
  }

  public String getContainerUrl(String azureContainerName) {
    return String.format("https://%s.%s/%s", azureAuthorization.getAzureStorageAccountName(),
        MicrosoftAzureAuthorization.AZURE_BLOB_STORE_URL_SUFFIX,
        StringUtils.trimToEmpty(azureContainerName));
  }

  public boolean containerExists(String azureContainerName)
          throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
    try {
      Map<String, String> containerProperties = getContainerProperties(azureContainerName);
      return containerProperties.containsKey("x-ms-blob-public-access") && StringUtils.equalsIgnoreCase("unlocked",
          containerProperties.getOrDefault("x-ms-lease-status", "INVALID"));
    } catch (MicrosoftAzureNotFoundException ex) {
      return false;
    }
  }

  public Map<String, String> getContainerProperties(String azureContainerName)
          throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException,
          MicrosoftAzureNotFoundException {
    String containerUrl = String.format("%s?%s", getContainerUrl(azureContainerName), "restype=container");
    String sasToken = azureAuthorization.generateAccountSASToken("r", "c",
        null, null, null, null);
    containerUrl = containerUrl + "&" + sasToken;

    try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
      HttpGet httpGet = new HttpGet(containerUrl);
      try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
        int code = response.getStatusLine().getStatusCode();
        Map<String, String> headersMap = Arrays.stream(response.getAllHeaders())
            .collect(Collectors.toMap(NameValuePair::getName, NameValuePair::getValue));
        switch (code) {
          case HttpStatus.SC_OK: // 200
            EntityUtils.consume(response.getEntity());
            break;
          case HttpStatus.SC_FORBIDDEN: // 403
            throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
                "Not allowed to read Azure storage container properties for container %s.",
                azureContainerName)));
          case HttpStatus.SC_NOT_FOUND: // 404
            throw new MicrosoftAzureNotFoundException(HttpUtils.formatResponseErrorString(response, String.format(
                "Azure storage container %s does not exists.", azureContainerName)));
          default:
            throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response, String.format(
                "Getting Azure storage container metadata failed with HTTP response code %d for container %s.",
                code, azureContainerName)));
        }
        return headersMap;
      }
    }
  }

  public void createContainer(String azureContainerName)
          throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
    if (containerExists(azureContainerName)) {
      return;
    }
    String containerUrl = String.format("%s?%s", getContainerUrl(azureContainerName), "restype=container");
    String sasToken = azureAuthorization.generateAccountSASToken("w", "c", null, null, null, null);
    containerUrl = containerUrl + "&" + sasToken;
    try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
      HttpPut httpPut = new HttpPut(containerUrl);
      httpPut.addHeader("x-ms-blob-public-access", "blob");
      try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
        int code = response.getStatusLine().getStatusCode();
        switch (code) {
          case HttpStatus.SC_CREATED: // 201
            EntityUtils.consume(response.getEntity());
            break;
          case HttpStatus.SC_FORBIDDEN: // 403
            throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
                "Not allowed to read Azure storage container properties for container %s.", azureContainerName)));

          default:
            throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response, String.format(
                "Creating Azure storage container %s failed with HTTP response code %d.", azureContainerName, code)));
        }
      }
    }
  }

  public String uploadFile(File trackFile, String azureContainerName, String azureBlobPath, String azureBlobName)
          throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
    String containerUrl = getContainerUrl(azureContainerName);
    String blobPath = Paths.get(StringUtils.trimToEmpty(azureBlobPath), StringUtils.trimToEmpty(azureBlobName))
        .normalize().toString();
    URL blobUrl = new URL(containerUrl + "/" + blobPath);
    int blockSize = 100000000; // 100MB
    String sasToken = azureAuthorization.generateServiceSasToken("w", null, null, blobUrl.getPath(), "b");
    try (FileInputStream trackStream = new FileInputStream(trackFile)) {
      try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
        List<String> blockIds = new ArrayList<>();
        // put blocks (file chunks)
        for (int iteration = 0; iteration * blockSize < trackFile.length(); iteration++) {
          String blockId = Base64.encodeBase64String(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));
          String putBlockUrl = blobUrl + "?comp=block&blockid="
              + URLEncoder.encode(blockId, StandardCharsets.UTF_8) + "&" + sasToken;
          HttpPut httpPut = new HttpPut(putBlockUrl);
          byte[] blockData = trackStream.readNBytes(blockSize);
          httpPut.setEntity(new ByteArrayEntity(blockData, ContentType.APPLICATION_OCTET_STREAM));
          try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
            int code = response.getStatusLine().getStatusCode();
            switch (code) {
              case HttpStatus.SC_CREATED: // 201
                blockIds.add(blockId);
                EntityUtils.consume(response.getEntity());
                break;
              case HttpStatus.SC_FORBIDDEN: // 403
                throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
                    "Not allowed to put block to Azure storage container %s.", azureContainerName)));
              default:
                throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response,
                    String.format("Putting block to Azure storage container %s failed with HTTP response code %d. ",
                        azureContainerName, code)));
            }
          }
        }
        // commit block list
        StringBuffer blockList = new StringBuffer();
        blockList.append("<?xml version=\"1.0\" encoding=\"utf-8\"?>");
        blockList.append("<BlockList>");
        for (String blockId : blockIds) {
          blockList.append("<Uncommitted>");
          blockList.append(blockId);
          blockList.append("</Uncommitted>");
        }
        blockList.append("</BlockList>");
        String putBlockListUrl = blobUrl + "?comp=blocklist&" + sasToken;
        HttpPut httpPut = new HttpPut(putBlockListUrl);
        httpPut.setEntity(new StringEntity(blockList.toString(),
            ContentType.create("application/xml", StandardCharsets.UTF_8)));
        try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
          int code = response.getStatusLine().getStatusCode();
          switch (code) {
            case HttpStatus.SC_CREATED: // 201
              EntityUtils.consume(response.getEntity());
              break;
            case HttpStatus.SC_FORBIDDEN: // 403
              throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
                  "Not allowed to put block list to Azure storage container %s.", azureContainerName)));
            default:
              throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response,
                  String.format("Putting block list to Azure storage container %s failed with HTTP response code %d.",
                      azureContainerName, code)));
          }
        }
      }
    }
    return blobUrl.toString();
  }

  public void deleteFile(URL fileUrl)
          throws IOException, MicrosoftAzureNotAllowedException, MicrosoftAzureStorageClientException {
    String sasToken = azureAuthorization.generateServiceSasToken("dy", null, null, fileUrl.getPath(), "b");
    String deleteUrl = String.format("https://%s%s?%s", fileUrl.getHost(), fileUrl.getPath(), sasToken);
    try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
      HttpDelete httpDelete = new HttpDelete(deleteUrl);
      try (CloseableHttpResponse response = httpClient.execute(httpDelete)) {
        int code = response.getStatusLine().getStatusCode();
        String responseString = "";
        if (response.getEntity() != null) {
          responseString = EntityUtils.toString(response.getEntity());
        }
        switch (code) {
          case HttpStatus.SC_ACCEPTED:  // 202
          case HttpStatus.SC_NOT_FOUND: // 404
            break;
          case HttpStatus.SC_FORBIDDEN: // 403
            throw new MicrosoftAzureNotAllowedException(String.format("Not allowed to delete storage blob %s. "
                + "Microsoft Azure Storage Service response: %s", httpDelete.getURI().toASCIIString(), responseString));
          default:
            throw new MicrosoftAzureStorageClientException(String.format("Deleting Azure storage blob '%s' failed "
                    + "with HTTP response code %d. Microsoft Azure Storage Service response: %s",
                httpDelete.getURI().toASCIIString(), code, responseString));
        }
      }
    }
  }
}