1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.opencastproject.caption.converters;
22
23 import org.opencastproject.caption.api.Caption;
24 import org.opencastproject.caption.api.CaptionConverter;
25 import org.opencastproject.caption.api.CaptionConverterException;
26 import org.opencastproject.caption.api.IllegalTimeFormatException;
27 import org.opencastproject.caption.api.Time;
28 import org.opencastproject.caption.impl.CaptionImpl;
29 import org.opencastproject.caption.impl.TimeImpl;
30 import org.opencastproject.mediapackage.MediaPackageElement;
31 import org.opencastproject.mediapackage.MediaPackageElement.Type;
32
33 import org.json.simple.JSONArray;
34 import org.json.simple.JSONObject;
35 import org.json.simple.parser.JSONParser;
36 import org.osgi.service.component.annotations.Component;
37 import org.slf4j.Logger;
38 import org.slf4j.LoggerFactory;
39
40 import java.io.IOException;
41 import java.io.InputStream;
42 import java.io.InputStreamReader;
43 import java.io.OutputStream;
44 import java.text.NumberFormat;
45 import java.util.ArrayList;
46 import java.util.List;
47 import java.util.Locale;
48
49 @Component(
50 immediate = true,
51 service = { CaptionConverter.class },
52 property = {
53 "service.description=IBM Watson caption converter",
54 "caption.format=ibm-watson"
55 }
56 )
57 public class IBMWatsonCaptionConverter implements CaptionConverter {
58
59
60 private static final Logger logger = LoggerFactory.getLogger(IBMWatsonCaptionConverter.class);
61
62 private static final int LINE_SIZE = 100;
63
64 @Override
65 public List<Caption> importCaption(InputStream inputStream, String language) throws CaptionConverterException {
66 List<Caption> captionList = new ArrayList<Caption>();
67 JSONParser jsonParser = new JSONParser();
68
69 try {
70 JSONObject resultsObj = (JSONObject) jsonParser.parse(new InputStreamReader(inputStream));
71 String jobId = "Unknown";
72 if (resultsObj.get("id") != null)
73 jobId = (String) resultsObj.get("id");
74
75
76 if (resultsObj.get("warnings") != null) {
77 JSONArray warningsArray = (JSONArray) resultsObj.get("warnings");
78 if (warningsArray != null) {
79 for (Object w : warningsArray)
80 logger.warn("Warning from Speech-To-Text service: {}", w);
81 }
82 }
83
84 JSONArray outerResultsArray = (JSONArray) resultsObj.get("results");
85 JSONObject obj = (JSONObject) outerResultsArray.get(0);
86 JSONArray resultsArray = (JSONArray) obj.get("results");
87
88 resultsLoop:
89 for (int i = 0; i < resultsArray.size(); i++) {
90 JSONObject resultElement = (JSONObject) resultsArray.get(i);
91
92 if (!(Boolean) resultElement.get("final"))
93 continue;
94
95 JSONArray alternativesArray = (JSONArray) resultElement.get("alternatives");
96 if (alternativesArray != null && alternativesArray.size() > 0) {
97 JSONObject alternativeElement = (JSONObject) alternativesArray.get(0);
98 String transcript = (String) alternativeElement.get("transcript");
99 if (transcript != null) {
100 JSONArray timestampsArray = (JSONArray) alternativeElement.get("timestamps");
101 if (timestampsArray == null || timestampsArray.size() == 0) {
102 logger.warn("Could not build caption object for job {}, result index {}: timestamp data not found",
103 jobId, i);
104 continue;
105 }
106
107 String[] words = transcript.split("\\s+");
108 StringBuffer line = new StringBuffer();
109 int indexFirst = -1;
110 int indexLast = -1;
111 for (int j = 0; j < words.length; j++) {
112 if (indexFirst == -1)
113 indexFirst = j;
114 line.append(words[j]);
115 line.append(" ");
116 if (line.length() >= LINE_SIZE || j == words.length - 1) {
117 indexLast = j;
118
119 double start = -1;
120 double end = -1;
121 if (indexLast < timestampsArray.size()) {
122
123 JSONArray wordTsArray = (JSONArray) timestampsArray.get(indexFirst);
124 if (wordTsArray.size() == 3)
125 start = NumberFormat.getInstance(Locale.US).parse(removeEndCharacter((wordTsArray.get(1).toString()),
126 "s")).doubleValue();
127
128 wordTsArray = (JSONArray) timestampsArray.get(indexLast);
129 if (wordTsArray.size() == 3)
130 end = NumberFormat.getInstance(Locale.US).parse(removeEndCharacter((wordTsArray.get(2).toString()),
131 "s")).doubleValue();
132 }
133 if (start == -1 || end == -1) {
134 logger.warn("Could not build caption object for job {}, result index {}: start/end times not found",
135 jobId, i);
136 continue resultsLoop;
137 }
138
139 String[] captionLines = new String[1];
140 captionLines[0] = line.toString().replace("%HESITATION", "...");
141 captionList.add(new CaptionImpl(buildTime((long) (start * 1000)), buildTime((long) (end * 1000)),
142 captionLines));
143 indexFirst = -1;
144 indexLast = -1;
145 line.setLength(0);
146 }
147 }
148 }
149 }
150 }
151 } catch (Exception e) {
152 logger.warn("Error when parsing IBM Watson transcriptions result", e);
153 throw new CaptionConverterException(e);
154 }
155
156 return captionList;
157 }
158
159 @Override
160 public void exportCaption(OutputStream outputStream, List<Caption> captions, String language) throws IOException {
161 throw new UnsupportedOperationException();
162 }
163
164 @Override
165 public String[] getLanguageList(InputStream inputStream) throws CaptionConverterException {
166 throw new UnsupportedOperationException();
167 }
168
169 @Override
170 public String getExtension() {
171 return "json";
172 }
173
174 @Override
175 public Type getElementType() {
176 return MediaPackageElement.Type.Attachment;
177 }
178
179 private Time buildTime(long ms) throws IllegalTimeFormatException {
180 int h = (int) (ms / 3600000L);
181 int m = (int) ((ms % 3600000L) / 60000L);
182 int s = (int) ((ms % 60000L) / 1000L);
183 ms = (int) (ms % 1000);
184
185 return new TimeImpl(h, m, s, (int) ms);
186 }
187
188 private String removeEndCharacter(String str, String end) {
189 if (str.endsWith(end)) {
190 str = str.replace(end, "");
191 }
192 return str;
193 }
194
195 }