1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.opencastproject.statistics.provider.matomo.provider;
23
24 import org.opencastproject.statistics.api.DataResolution;
25 import org.opencastproject.statistics.api.ResourceType;
26 import org.opencastproject.statistics.api.TimeSeries;
27 import org.opencastproject.statistics.provider.matomo.StatisticsProviderMatomoService;
28
29 import com.google.gson.JsonArray;
30 import com.google.gson.JsonElement;
31 import com.google.gson.JsonObject;
32 import com.google.gson.JsonParseException;
33 import com.google.gson.JsonParser;
34
35 import org.slf4j.Logger;
36 import org.slf4j.LoggerFactory;
37
38 import java.net.URI;
39 import java.net.URLEncoder;
40 import java.net.http.HttpClient;
41 import java.net.http.HttpRequest;
42 import java.net.http.HttpResponse;
43 import java.nio.charset.StandardCharsets;
44 import java.time.Instant;
45 import java.time.ZoneId;
46 import java.time.format.DateTimeFormatter;
47 import java.util.ArrayList;
48 import java.util.Arrays;
49 import java.util.HashMap;
50 import java.util.List;
51 import java.util.Map;
52 import java.util.Objects;
53 import java.util.concurrent.ConcurrentHashMap;
54
55 public class BatchMatomoRequest {
56 private static final Logger logger = LoggerFactory.getLogger(BatchMatomoRequest.class);
57
58 private final StatisticsProviderMatomoService service;
59 private final String method;
60 private final Map<String, MatomoTimeSeriesStatisticsProvider> providers;
61 private final Map<String, String> aggregationVariables;
62 private final Map<String, String> aggregationTypes;
63
64
65 private static final long DEFAULT_CACHE_DURATION_MS = 5 * 60 * 1000;
66
67 private static final long CLEANUP_INTERVAL_MS = 60 * 1000;
68
69 private final long cacheDurationMs;
70 private final Map<CacheKey, CacheEntry> resultCache;
71 private long lastCleanupTime;
72
73
74 private static class CacheEntry {
75 private final JsonObject apiResponse;
76 private final long timestamp;
77
78 CacheEntry(JsonObject apiResponse) {
79 this.apiResponse = apiResponse;
80 this.timestamp = System.currentTimeMillis();
81 }
82
83 boolean isExpired(long maxAgeMs) {
84 return System.currentTimeMillis() - timestamp > maxAgeMs;
85 }
86
87 JsonObject getApiResponse() {
88 return apiResponse;
89 }
90 }
91
92
93 private static class CacheKey {
94 private final String resourceId;
95 private final Instant from;
96 private final Instant to;
97 private final String period;
98 private final String siteId;
99 private final String dimensionId;
100 private final ZoneId zoneId;
101 private final DataResolution resolution;
102
103 CacheKey(String resourceId, Instant from, Instant to, String period,
104 String siteId, String dimensionId, ZoneId zoneId, DataResolution resolution) {
105 this.resourceId = resourceId;
106 this.from = from;
107 this.to = to;
108 this.period = period;
109 this.siteId = siteId;
110 this.dimensionId = dimensionId;
111 this.zoneId = zoneId;
112 this.resolution = resolution;
113 }
114
115
116 @Override
117 public boolean equals(Object o) {
118 if (this == o) {
119 return true;
120 }
121 if (o == null || getClass() != o.getClass()) {
122 return false;
123 }
124 CacheKey cacheKey = (CacheKey) o;
125
126 return Objects.equals(resourceId, cacheKey.resourceId)
127 && Objects.equals(from, cacheKey.from)
128 && Objects.equals(to, cacheKey.to)
129 && Objects.equals(period, cacheKey.period)
130 && Objects.equals(siteId, cacheKey.siteId)
131 && Objects.equals(dimensionId, cacheKey.dimensionId)
132 && Objects.equals(zoneId, cacheKey.zoneId)
133 && resolution == cacheKey.resolution;
134 }
135
136
137 @Override
138 public int hashCode() {
139 return Objects.hash(resourceId, from, to, period, siteId, dimensionId, zoneId, resolution);
140 }
141 }
142
143 public BatchMatomoRequest(StatisticsProviderMatomoService service, String method) {
144 this(service, method, DEFAULT_CACHE_DURATION_MS);
145 }
146
147 public BatchMatomoRequest(StatisticsProviderMatomoService service, String method, long cacheDurationMs) {
148 this.service = service;
149 this.method = method;
150 this.providers = new HashMap<>();
151 this.aggregationVariables = new HashMap<>();
152 this.aggregationTypes = new HashMap<>();
153 this.resultCache = new ConcurrentHashMap<>();
154 this.cacheDurationMs = cacheDurationMs;
155 this.lastCleanupTime = System.currentTimeMillis();
156 }
157
158
159 private void cleanupExpiredEntries() {
160 long currentTime = System.currentTimeMillis();
161 if (currentTime - lastCleanupTime > CLEANUP_INTERVAL_MS) {
162 resultCache.entrySet().removeIf(entry -> entry.getValue().isExpired(cacheDurationMs));
163 lastCleanupTime = currentTime;
164 logger.debug("Performed cache cleanup. Cache size: {}", resultCache.size());
165 }
166 }
167
168 public void addProvider(
169 MatomoTimeSeriesStatisticsProvider provider,
170 String aggregationVariable,
171 String aggregationType) {
172 this.providers.put(provider.getId(), provider);
173 this.aggregationVariables.put(provider.getId(), aggregationVariable);
174 this.aggregationTypes.put(provider.getId(), aggregationType);
175 }
176
177
178 private Map<String, TimeSeries> processApiResponse(
179 JsonObject apiResponse,
180 String resourceId,
181 DataResolution resolution) {
182
183 DateTimeFormatter outputFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");
184 Map<String, TimeSeries> results = new HashMap<>();
185
186 for (Map.Entry<String, MatomoTimeSeriesStatisticsProvider> entry : providers.entrySet()) {
187 String providerId = entry.getKey();
188 String aggregationVariable = aggregationVariables.get(providerId);
189
190 List<String> labels = new ArrayList<>();
191 List<Double> values = new ArrayList<>();
192
193 for (String matomoDateStr : apiResponse.keySet()) {
194
195 String date = matomoDateStr;
196 if (matomoDateStr.length() == 4) {
197 date = matomoDateStr + "-01-01T00:00:00Z";
198 } else if (matomoDateStr.length() == 7) {
199 date = matomoDateStr + "-01T00:00:00Z";
200 } else if (matomoDateStr.length() == 10) {
201 date = matomoDateStr + "T00:00:00Z";
202 }
203
204 try {
205 outputFormatter.parse(date);
206 labels.add(date);
207 } catch (Exception e) {
208 logger.warn("Unexpected date format {}: {}", matomoDateStr, e.getMessage());
209 continue;
210 }
211
212 JsonObject matomoData = null;
213 if (apiResponse.get(matomoDateStr).isJsonArray()) {
214
215
216
217
218 JsonArray dataArray = apiResponse.get(matomoDateStr).getAsJsonArray();
219 if (dataArray != null && dataArray.size() > 0) {
220 if (entry.getValue().getResourceType() != ResourceType.ORGANIZATION) {
221 for (JsonElement element : dataArray) {
222 JsonObject dataElement = element.getAsJsonObject();
223 if (dataElement.has("label") && dataElement.get("label").getAsString().equals(resourceId)) {
224 matomoData = element.getAsJsonObject();
225 break;
226 }
227 }
228 } else {
229 matomoData = dataArray.get(0).getAsJsonObject();
230 }
231 }
232 } else if (apiResponse.get(matomoDateStr).isJsonObject()) {
233 matomoData = apiResponse.get(matomoDateStr).getAsJsonObject();
234 }
235
236 if (matomoData != null && matomoData.has(aggregationVariable)) {
237 logger.debug("Matomo data for provider '{}' [{}, {}: {}]",
238 providerId,
239 matomoDateStr,
240 aggregationVariable,
241 matomoData.get(aggregationVariable));
242 values.add(matomoData.get(aggregationVariable).getAsDouble());
243 } else {
244
245 values.add(0.0);
246 }
247 }
248
249
250 final Double total = "SUM".equalsIgnoreCase(aggregationTypes.get(providerId))
251 ? values.stream().mapToDouble(v -> v).sum()
252 : null;
253
254 TimeSeries timeSeries = new TimeSeries(labels, values, total);
255 results.put(providerId, timeSeries);
256 }
257 return results;
258 }
259
260
261 public Map<String, TimeSeries> executeRequest(
262 String resourceId,
263 Instant from,
264 Instant to,
265 String matomoPeriod,
266 String siteId,
267 String dimensionId,
268 ZoneId zoneId,
269 DataResolution resolution) {
270
271
272 cleanupExpiredEntries();
273
274
275 CacheKey cacheKey = new CacheKey(resourceId, from, to, matomoPeriod,
276 siteId, dimensionId, zoneId, resolution);
277
278
279 CacheEntry cacheEntry = resultCache.get(cacheKey);
280 if (cacheEntry != null) {
281 if (!cacheEntry.isExpired(cacheDurationMs)) {
282
283 logger.debug("Using cached Matomo API response for resourceId: {}, method: {}", resourceId, method);
284 return processApiResponse(cacheEntry.getApiResponse(), resourceId, resolution);
285 } else {
286
287 resultCache.remove(cacheKey);
288 }
289 }
290
291
292 JsonObject apiResponse = null;
293 DateTimeFormatter inputFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
294 Map<String, TimeSeries> results = new HashMap<>();
295 String url = "";
296 String matomoApiUrl = service.getMatomoApiUrl();
297 String matomoApiToken = service.getMatomoApiToken();
298
299 try {
300 if (matomoApiUrl == null || matomoApiUrl.isEmpty() || matomoApiToken == null || matomoApiToken.isEmpty()) {
301 logger.error("Matomo API parameters are missing in config file, skip requesting data from Matomo API.");
302 } else {
303 url = matomoApiUrl + "/index.php?module=API"
304 + "&format=json&filter_limit=-1&expanded=1"
305 + "&idSite=" + siteId
306 + "&method=" + method
307 + "&date="
308 + from.atZone(zoneId).toLocalDate().format(inputFormatter) + ","
309 + to.atZone(zoneId).toLocalDate().format(inputFormatter)
310 + "&period=" + matomoPeriod;
311
312 if (dimensionId != null) {
313 url += "&idDimension=" + dimensionId;
314 }
315
316 if (!providers.isEmpty()
317 && providers.values().iterator().next().getResourceType() != ResourceType.ORGANIZATION
318 ) {
319 url += "&label=" + resourceId;
320 }
321
322 String requestBody = "token_auth=" + URLEncoder.encode(matomoApiToken, StandardCharsets.UTF_8.name());
323 logger.debug("Sending Matomo API request for resourceId: {}, method: {}: {}", resourceId, method, url);
324 HttpClient client = HttpClient.newHttpClient();
325 HttpRequest request = HttpRequest.newBuilder()
326 .uri(URI.create(url))
327 .header("Content-Type", "application/x-www-form-urlencoded")
328 .POST(HttpRequest.BodyPublishers.ofString(requestBody))
329 .build();
330
331 HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
332
333 if (response.statusCode() >= 300) {
334 logger.error("Matomo API unexpected status code " + response.statusCode() + ": " + url);
335 } else {
336 String responseBody = response.body();
337
338 try {
339 JsonElement rootElement = JsonParser.parseString(responseBody);
340 if (rootElement.isJsonObject()) {
341 apiResponse = rootElement.getAsJsonObject();
342 } else {
343 logger.error("Unexpected JSON format: Root element is not a JSON object.");
344 }
345 } catch (JsonParseException e) {
346 logger.error("Error parsing Matomo API response {}: {}", url, e.getMessage());
347 }
348 }
349 }
350 } catch (Exception e) {
351 logger.error("Error connecting to Matomo API {}: {}", url, e.getMessage());
352 }
353
354 if (apiResponse == null || apiResponse.entrySet().isEmpty()) {
355
356 logger.error("Because of errors connecting to Matomo returning empty TimeSeries for providers {}.",
357 providers.keySet());
358 for (Map.Entry<String, MatomoTimeSeriesStatisticsProvider> entry : providers.entrySet()) {
359 String providerId = entry.getKey();
360 TimeSeries timeSeries = new TimeSeries(
361 new ArrayList<>(Arrays.asList(from.toString())),
362 new ArrayList<>(Arrays.asList(0.0)),
363 null);
364 results.put(providerId, timeSeries);
365 }
366 } else {
367
368 logger.debug("Write Matomo API response to cache and process results.");
369 resultCache.put(cacheKey, new CacheEntry(apiResponse));
370 results = processApiResponse(apiResponse, resourceId, resolution);
371 }
372 return results;
373 }
374 }