View Javadoc
1   /*
2    * Licensed to The Apereo Foundation under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional
4    * information regarding copyright ownership.
5    *
6    *
7    * The Apereo Foundation licenses this file to you under the Educational
8    * Community License, Version 2.0 (the "License"); you may not use this file
9    * except in compliance with the License. You may obtain a copy of the License
10   * at:
11   *
12   *   http://opensource.org/licenses/ecl2.txt
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
17   * License for the specific language governing permissions and limitations under
18   * the License.
19   *
20   */
21  package org.opencastproject.assetmanager.impl.query;
22  
23  import static com.entwinemedia.fn.Stream.$;
24  import static java.lang.String.format;
25  
26  import org.opencastproject.assetmanager.api.query.ADeleteQuery;
27  import org.opencastproject.assetmanager.api.query.Predicate;
28  import org.opencastproject.assetmanager.api.storage.AssetStore;
29  import org.opencastproject.assetmanager.api.storage.DeletionSelector;
30  import org.opencastproject.assetmanager.impl.AssetManagerImpl;
31  import org.opencastproject.assetmanager.impl.RuntimeTypes;
32  import org.opencastproject.assetmanager.impl.VersionImpl;
33  import org.opencastproject.assetmanager.impl.persistence.Conversions;
34  import org.opencastproject.assetmanager.impl.persistence.EntityPaths;
35  import org.opencastproject.assetmanager.impl.persistence.QPropertyDto;
36  import org.opencastproject.assetmanager.impl.persistence.QSnapshotDto;
37  import org.opencastproject.util.data.Function;
38  
39  import com.entwinemedia.fn.Fn;
40  import com.entwinemedia.fn.data.SetB;
41  import com.mysema.query.Tuple;
42  import com.mysema.query.jpa.JPASubQuery;
43  import com.mysema.query.jpa.impl.JPADeleteClause;
44  import com.mysema.query.jpa.impl.JPAQueryFactory;
45  import com.mysema.query.support.Expressions;
46  import com.mysema.query.types.EntityPath;
47  import com.mysema.query.types.expr.BooleanExpression;
48  
49  import org.slf4j.Logger;
50  import org.slf4j.LoggerFactory;
51  
52  import java.util.Collections;
53  import java.util.List;
54  import java.util.Set;
55  
56  public abstract class AbstractADeleteQuery implements ADeleteQuery, DeleteQueryContributor, EntityPaths {
57    private static final Logger logger = LoggerFactory.getLogger(AbstractADeleteQuery.class);
58  
59    private AssetManagerImpl am;
60    private String owner;
61  
62    public AbstractADeleteQuery(AssetManagerImpl am, String owner) {
63      this.am = am;
64      this.owner = owner;
65    }
66  
67    @Override public ADeleteQuery name(final String queryName) {
68      return new AbstractADeleteQuery(am, owner) {
69        @Override public DeleteQueryContribution contributeDelete(String owner) {
70          final DeleteQueryContribution cParent = AbstractADeleteQuery.this.contributeDelete(owner);
71          return DeleteQueryContribution.mk(cParent).name(queryName);
72        }
73      };
74    }
75  
76    @Override public ADeleteQuery where(final Predicate predicate) {
77      return new AbstractADeleteQuery(am, owner) {
78        @Override public DeleteQueryContribution contributeDelete(String owner) {
79          final DeleteQueryContribution cParent = AbstractADeleteQuery.this.contributeDelete(owner);
80          final DeleteQueryContribution cPredicate = RuntimeTypes.convert(predicate).contributeDelete(owner);
81          return DeleteQueryContribution.mk()
82                  .from(cParent.from.append(cPredicate.from))
83                  .targetPredicate(cParent.targetPredicate)
84                  .where(JpaFns.allOf(cParent.where, cPredicate.where));
85        }
86  
87        @Override public String toString() {
88          return "where " + predicate;
89        }
90      };
91    }
92  
93    public long run(DeleteEpisodeHandler deleteEpisodeHandler) {
94      // run query and map the result to records
95      final long startTime = System.nanoTime();
96      // resolve AST
97      final DeleteQueryContribution c = contributeDelete(owner);
98      // run all queries in a single transaction
99      final DeletionResult deletion = am.getDatabase().run(new Function<JPAQueryFactory, DeletionResult>() {
100       @Override public DeletionResult apply(final JPAQueryFactory jpa) {
101         return runQueries(jpa, c);
102       }
103     });
104     logger.debug("Pure query ms " + (System.nanoTime() - startTime) / 1000000);
105     // delete from store
106     for (Tuple t : deletion.deletedSnapshots) {
107       // all three t.get(..) calls won't return null since the database fields are not null.
108       final String orgId = t.get(Q_SNAPSHOT.organizationId);
109       final String mpId = t.get(Q_SNAPSHOT.mediaPackageId);
110       final VersionImpl version = Conversions.toVersion(t.get(Q_SNAPSHOT.version));
111       final DeletionSelector deletionSelector = DeletionSelector.delete(orgId, mpId, version);
112       am.getLocalAssetStore().delete(deletionSelector);
113       for (AssetStore as : am.getRemoteAssetStores()) {
114         as.delete(deletionSelector);
115       }
116     }
117     for (String mpId : deletion.deletedEpisodes) {
118       deleteEpisodeHandler.handleDeletedEpisode(mpId);
119     }
120     final long searchTime = (System.nanoTime() - startTime) / 1000000;
121     logger.debug("Complete query ms " + searchTime);
122     return deletion.deletedItemsCount;
123   }
124 
125   /** Run this in a transaction. */
126   private DeletionResult runQueries(JPAQueryFactory jpa, DeleteQueryContribution c) {
127     // # create Querydsl delete clause
128     // # from
129     // put into a set to remove duplicates
130     final EntityPath<?> from;
131     {
132       final Set<EntityPath<?>> f = c.from.toSet(SetB.MH);
133       if (f.size() == 1) {
134         from = $(f).head2();
135       } else {
136         throw new RuntimeException("Only one entity is allowed in the from clause");
137       }
138     }
139     //
140     if (from instanceof QSnapshotDto) {
141       // from Snapshot
142       //
143       final BooleanExpression where = Expressions.allOf(
144               c.targetPredicate.orNull(),
145               c.where.apply(Q_SNAPSHOT));
146       // get snapshots to delete
147       // TODO ATTENTION: this query has the potential to yield a massive amount of elements
148       // return the list of snapshots to delete them outside the transaction since
149       // it may take a while.
150       final List<Tuple> deletedSnapshots = jpa.query()
151               .from(Q_SNAPSHOT)
152               .where(where)
153               .list(Q_SNAPSHOT.organizationId, Q_SNAPSHOT.mediaPackageId, Q_SNAPSHOT.version);
154 // <BLOCK>
155 // TODO database only approach to determine deleted episodes
156 // TODO does not run with H2 so unit tests break
157       /*
158 SELECT
159   e.mediapackage_id,
160   count(*) AS v
161 FROM oc_assets_snapshot e
162 GROUP BY e.mediapackage_id
163 HAVING v = (SELECT count(*)
164             FROM oc_assets_snapshot e2
165             WHERE e.mediapackage_id = e2.mediapackage_id
166                   AND
167                   -- delete where clause
168                   (e2.version = 2 OR e2.mediapackage_id = '24ec925e-ea57-43a5-a7bb-58dc5aae54dd')
169             GROUP BY mediapackage_id);
170        */
171 //      final QSnapshotDto e2 = new QSnapshotDto("eee");
172 //      final List<String> deletedSnapshots = jpa.query()
173 //              .from(e2)
174 //              .groupBy(e2.mediaPackageId)
175 //              .having(e2.count().eq(
176 //                      jpa.subQuery()
177 //                              .from(Q_SNAPSHOT)
178 //                              .where(Q_SNAPSHOT.mediaPackageId.eq(e2.mediaPackageId).and(where))
179 //                              .groupBy(Q_SNAPSHOT.mediaPackageId)
180 //                              .count()))
181 //              .list(e2.mediaPackageId);
182 // </BLOCK>
183       // main delete query
184       final JPADeleteClause qMain = jpa.delete(Q_SNAPSHOT).where(where);
185       am.getDatabase().logDelete(formatQueryName(c.name, "main"), qMain);
186       final long deletedItems = qMain.execute();
187       // <BLOCK>
188       // TODO Bad solution. Yields all media package IDs which can easily be thousands
189       // TODO The above SQL solution does not work with H2 so I suspect the query is not 100% clean
190       // TODO Rework the query and replace this code.
191       // calculate deleted episodes, i.e. where all snapshots have been deleted
192       final Set<String> deletedEpisodes;
193       {
194         final List<String> remainingSnapshots = jpa.query()
195                 .from(Q_SNAPSHOT)
196                 .distinct()
197                 .list(Q_SNAPSHOT.mediaPackageId);
198         final Set<String> d = $(deletedSnapshots).map(new Fn<Tuple, String>() {
199           @Override public String apply(Tuple tuple) {
200             return tuple.get(Q_SNAPSHOT.mediaPackageId);
201           }
202         }).toSet(SetB.MH);
203         d.removeAll(remainingSnapshots);
204         deletedEpisodes = Collections.unmodifiableSet(d);
205       }
206       // </BLOCK>
207       return new DeletionResult(deletedItems, deletedSnapshots, deletedEpisodes);
208     } else if (from instanceof QPropertyDto) {
209       // from Property
210       //
211       final BooleanExpression where;
212       {
213         final BooleanExpression w = c.where.apply(Q_PROPERTY);
214         if (w != null) {
215           /* The original sub query used an "ON" clause to filter the join by mediapackage id [1].
216              Unfortunately Eclipse link drops this clause completely when transforming the query
217              into SQL. It creates a cross join instead of the inner join, which is perfectly legal
218              if the "ON" clause would be moved to the "WHERE" clause.
219              The example [2] shows that neither an "ON" clause nor an additional "WHERE" predicate is generated.
220 
221              [1]
222              new JPASubQuery()
223                 .from(Q_PROPERTY)
224                 .join(Q_SNAPSHOT) <- inner join
225                 .on(Q_PROPERTY.mediaPackageId.eq(Q_SNAPSHOT.mediaPackageId)) <- dropped by Eclipse link
226                 .where(Q_PROPERTY.mediaPackageId.eq(Q_SNAPSHOT.mediaPackageId).and(w))
227                 .distinct()
228                 .list(Q_PROPERTY.mediaPackageId)
229 
230              [2]
231              SELECT DISTINCT t1.mediapackage_id FROM oc_assets_snapshot t2, oc_assets_properties t1
232                  WHERE (t2.organization_id = ?)
233            */
234           where = Q_PROPERTY.mediaPackageId.in(
235               new JPASubQuery()
236                   .from(Q_PROPERTY)
237                   .join(Q_SNAPSHOT)
238                   // move the join condition from the "ON" clause (mediapackage_id) to the
239                   // where clause. Find an explanation above.
240                   .where(Q_PROPERTY.mediaPackageId.eq(Q_SNAPSHOT.mediaPackageId).and(w))
241                   .distinct()
242                   .list(Q_PROPERTY.mediaPackageId));
243         } else {
244           where = null;
245         }
246       }
247       final JPADeleteClause qProperties = jpa.delete(from).where(Expressions.allOf(c.targetPredicate.orNull(), where));
248       am.getDatabase().logDelete(formatQueryName(c.name, "main"), qProperties);
249       final long deletedItems = qProperties.execute();
250       return new DeletionResult(deletedItems, Collections.<Tuple>emptyList(), Collections.<String>emptySet());
251     } else {
252       // from contains an unsupported entity
253       throw new RuntimeException("[Bug]");
254     }
255   }
256 
257   @Override public long run() {
258     return run(DELETE_EPISODE_HANDLER);
259   }
260 
261   private static String formatQueryName(String name, String subQueryName) {
262     return format("[%s] [%s]", name, subQueryName);
263   }
264 
265   /**
266    * Call {@link #run(DeleteEpisodeHandler)} with a deletion handler to get notified about deletions.
267    */
268   public interface DeleteEpisodeHandler {
269 
270     void handleDeletedEpisode(String mpId);
271   }
272 
273   public static final DeleteEpisodeHandler DELETE_EPISODE_HANDLER = new DeleteEpisodeHandler() {
274 
275     @Override public void handleDeletedEpisode(String mpId) {
276     }
277   };
278 
279   public final class DeletionResult {
280     // CHECKSTYLE:OFF
281     public final long deletedItemsCount;
282     public final List<Tuple> deletedSnapshots;
283     public final Set<String> deletedEpisodes;
284     // CHECKSTYLE:ON
285 
286     public DeletionResult(
287             long deletedItemsCount, List<Tuple> deletedSnapshots, Set<String> deletedEpisodes) {
288       this.deletedItemsCount = deletedItemsCount;
289       this.deletedSnapshots = deletedSnapshots;
290       this.deletedEpisodes = deletedEpisodes;
291     }
292   }
293 }