1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.transcription.microsoft.azure;
23
24 import org.apache.commons.codec.binary.Base64;
25 import org.apache.commons.lang3.StringUtils;
26 import org.apache.http.HttpStatus;
27 import org.apache.http.NameValuePair;
28 import org.apache.http.client.methods.CloseableHttpResponse;
29 import org.apache.http.client.methods.HttpDelete;
30 import org.apache.http.client.methods.HttpGet;
31 import org.apache.http.client.methods.HttpPut;
32 import org.apache.http.entity.ByteArrayEntity;
33 import org.apache.http.entity.ContentType;
34 import org.apache.http.entity.StringEntity;
35 import org.apache.http.impl.client.CloseableHttpClient;
36 import org.apache.http.util.EntityUtils;
37 import org.slf4j.Logger;
38 import org.slf4j.LoggerFactory;
39
40 import java.io.File;
41 import java.io.FileInputStream;
42 import java.io.IOException;
43 import java.net.URL;
44 import java.net.URLEncoder;
45 import java.nio.charset.StandardCharsets;
46 import java.nio.file.Paths;
47 import java.util.ArrayList;
48 import java.util.Arrays;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.UUID;
52 import java.util.stream.Collectors;
53
54 public class MicrosoftAzureStorageClient {
55
56 private static final Logger logger = LoggerFactory.getLogger(MicrosoftAzureStorageClient.class);
57
58 private MicrosoftAzureAuthorization azureAuthorization;
59
60 public MicrosoftAzureStorageClient(MicrosoftAzureAuthorization azureAuthorization) {
61 this.azureAuthorization = azureAuthorization;
62 }
63
64 public String getContainerUrl(String azureContainerName) {
65 return String.format("https://%s.%s/%s", azureAuthorization.getAzureStorageAccountName(),
66 MicrosoftAzureAuthorization.AZURE_BLOB_STORE_URL_SUFFIX,
67 StringUtils.trimToEmpty(azureContainerName));
68 }
69
70 public boolean containerExists(String azureContainerName)
71 throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
72 try {
73 Map<String, String> containerProperties = getContainerProperties(azureContainerName);
74 return containerProperties.containsKey("x-ms-blob-public-access") && StringUtils.equalsIgnoreCase("unlocked",
75 containerProperties.getOrDefault("x-ms-lease-status", "INVALID"));
76 } catch (MicrosoftAzureNotFoundException ex) {
77 return false;
78 }
79 }
80
81 public Map<String, String> getContainerProperties(String azureContainerName)
82 throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException,
83 MicrosoftAzureNotFoundException {
84 String containerUrl = String.format("%s?%s", getContainerUrl(azureContainerName), "restype=container");
85 String sasToken = azureAuthorization.generateAccountSASToken("r", "c",
86 null, null, null, null);
87 containerUrl = containerUrl + "&" + sasToken;
88
89 try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
90 HttpGet httpGet = new HttpGet(containerUrl);
91 try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
92 int code = response.getStatusLine().getStatusCode();
93 Map<String, String> headersMap = Arrays.stream(response.getAllHeaders())
94 .collect(Collectors.toMap(NameValuePair::getName, NameValuePair::getValue));
95 switch (code) {
96 case HttpStatus.SC_OK:
97 EntityUtils.consume(response.getEntity());
98 break;
99 case HttpStatus.SC_FORBIDDEN:
100 throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
101 "Not allowed to read Azure storage container properties for container %s.",
102 azureContainerName)));
103 case HttpStatus.SC_NOT_FOUND:
104 throw new MicrosoftAzureNotFoundException(HttpUtils.formatResponseErrorString(response, String.format(
105 "Azure storage container %s does not exists.", azureContainerName)));
106 default:
107 throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response, String.format(
108 "Getting Azure storage container metadata failed with HTTP response code %d for container %s.",
109 code, azureContainerName)));
110 }
111 return headersMap;
112 }
113 }
114 }
115
116 public void createContainer(String azureContainerName)
117 throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
118 if (containerExists(azureContainerName)) {
119 return;
120 }
121 String containerUrl = String.format("%s?%s", getContainerUrl(azureContainerName), "restype=container");
122 String sasToken = azureAuthorization.generateAccountSASToken("w", "c", null, null, null, null);
123 containerUrl = containerUrl + "&" + sasToken;
124 try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
125 HttpPut httpPut = new HttpPut(containerUrl);
126 httpPut.addHeader("x-ms-blob-public-access", "blob");
127 try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
128 int code = response.getStatusLine().getStatusCode();
129 switch (code) {
130 case HttpStatus.SC_CREATED:
131 EntityUtils.consume(response.getEntity());
132 break;
133 case HttpStatus.SC_FORBIDDEN:
134 throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
135 "Not allowed to read Azure storage container properties for container %s.", azureContainerName)));
136
137 default:
138 throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response, String.format(
139 "Creating Azure storage container %s failed with HTTP response code %d.", azureContainerName, code)));
140 }
141 }
142 }
143 }
144
145 public String uploadFile(File trackFile, String azureContainerName, String azureBlobPath, String azureBlobName)
146 throws MicrosoftAzureStorageClientException, IOException, MicrosoftAzureNotAllowedException {
147 String containerUrl = getContainerUrl(azureContainerName);
148 String blobPath = Paths.get(StringUtils.trimToEmpty(azureBlobPath), StringUtils.trimToEmpty(azureBlobName))
149 .normalize().toString();
150 URL blobUrl = new URL(containerUrl + "/" + blobPath);
151 int blockSize = 100000000;
152 String sasToken = azureAuthorization.generateServiceSasToken("w", null, null, blobUrl.getPath(), "b");
153 try (FileInputStream trackStream = new FileInputStream(trackFile)) {
154 try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
155 List<String> blockIds = new ArrayList<>();
156
157 for (int iteration = 0; iteration * blockSize < trackFile.length(); iteration++) {
158 String blockId = Base64.encodeBase64String(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));
159 String putBlockUrl = blobUrl + "?comp=block&blockid="
160 + URLEncoder.encode(blockId, StandardCharsets.UTF_8) + "&" + sasToken;
161 HttpPut httpPut = new HttpPut(putBlockUrl);
162 byte[] blockData = trackStream.readNBytes(blockSize);
163 httpPut.setEntity(new ByteArrayEntity(blockData, ContentType.APPLICATION_OCTET_STREAM));
164 try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
165 int code = response.getStatusLine().getStatusCode();
166 switch (code) {
167 case HttpStatus.SC_CREATED:
168 blockIds.add(blockId);
169 EntityUtils.consume(response.getEntity());
170 break;
171 case HttpStatus.SC_FORBIDDEN:
172 throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
173 "Not allowed to put block to Azure storage container %s.", azureContainerName)));
174 default:
175 throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response,
176 String.format("Putting block to Azure storage container %s failed with HTTP response code %d. ",
177 azureContainerName, code)));
178 }
179 }
180 }
181
182 StringBuffer blockList = new StringBuffer();
183 blockList.append("<?xml version=\"1.0\" encoding=\"utf-8\"?>");
184 blockList.append("<BlockList>");
185 for (String blockId : blockIds) {
186 blockList.append("<Uncommitted>");
187 blockList.append(blockId);
188 blockList.append("</Uncommitted>");
189 }
190 blockList.append("</BlockList>");
191 String putBlockListUrl = blobUrl + "?comp=blocklist&" + sasToken;
192 HttpPut httpPut = new HttpPut(putBlockListUrl);
193 httpPut.setEntity(new StringEntity(blockList.toString(),
194 ContentType.create("application/xml", StandardCharsets.UTF_8)));
195 try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
196 int code = response.getStatusLine().getStatusCode();
197 switch (code) {
198 case HttpStatus.SC_CREATED:
199 EntityUtils.consume(response.getEntity());
200 break;
201 case HttpStatus.SC_FORBIDDEN:
202 throw new MicrosoftAzureNotAllowedException(HttpUtils.formatResponseErrorString(response, String.format(
203 "Not allowed to put block list to Azure storage container %s.", azureContainerName)));
204 default:
205 throw new MicrosoftAzureStorageClientException(HttpUtils.formatResponseErrorString(response,
206 String.format("Putting block list to Azure storage container %s failed with HTTP response code %d.",
207 azureContainerName, code)));
208 }
209 }
210 }
211 }
212 return blobUrl.toString();
213 }
214
215 public void deleteFile(URL fileUrl)
216 throws IOException, MicrosoftAzureNotAllowedException, MicrosoftAzureStorageClientException {
217 String sasToken = azureAuthorization.generateServiceSasToken("dy", null, null, fileUrl.getPath(), "b");
218 String deleteUrl = String.format("https://%s%s?%s", fileUrl.getHost(), fileUrl.getPath(), sasToken);
219 try (CloseableHttpClient httpClient = HttpUtils.makeHttpClient()) {
220 HttpDelete httpDelete = new HttpDelete(deleteUrl);
221 try (CloseableHttpResponse response = httpClient.execute(httpDelete)) {
222 int code = response.getStatusLine().getStatusCode();
223 String responseString = "";
224 if (response.getEntity() != null) {
225 responseString = EntityUtils.toString(response.getEntity());
226 }
227 switch (code) {
228 case HttpStatus.SC_ACCEPTED:
229 case HttpStatus.SC_NOT_FOUND:
230 break;
231 case HttpStatus.SC_FORBIDDEN:
232 throw new MicrosoftAzureNotAllowedException(String.format("Not allowed to delete storage blob %s. "
233 + "Microsoft Azure Storage Service response: %s", httpDelete.getURI().toASCIIString(), responseString));
234 default:
235 throw new MicrosoftAzureStorageClientException(String.format("Deleting Azure storage blob '%s' failed "
236 + "with HTTP response code %d. Microsoft Azure Storage Service response: %s",
237 httpDelete.getURI().toASCIIString(), code, responseString));
238 }
239 }
240 }
241 }
242 }