Source code

001/*
002*  Copyright 2016 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013*  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.workspaces.repository.maintenance;
017
018import java.lang.reflect.InvocationTargetException;
019import java.lang.reflect.Method;
020import java.sql.SQLException;
021import java.util.ArrayList;
022import java.util.Iterator;
023
024import javax.jcr.Node;
025import javax.jcr.RepositoryException;
026import javax.jcr.Session;
027import javax.jcr.SimpleCredentials;
028import javax.sql.DataSource;
029
030import org.apache.jackrabbit.api.management.MarkEventListener;
031import org.apache.jackrabbit.core.RepositoryContext;
032import org.apache.jackrabbit.core.data.DataIdentifier;
033import org.apache.jackrabbit.core.data.DataStore;
034import org.apache.jackrabbit.core.data.DataStoreException;
035import org.apache.jackrabbit.core.data.FileDataStore;
036import org.apache.jackrabbit.core.data.db.DbDataStore;
037import org.apache.jackrabbit.core.data.db.DerbyDataStore;
038import org.apache.jackrabbit.core.gc.GarbageCollector;
039import org.apache.jackrabbit.core.persistence.IterablePersistenceManager;
040import org.apache.jackrabbit.core.persistence.PersistenceManager;
041import org.apache.jackrabbit.core.util.db.ConnectionFactory;
042import org.apache.jackrabbit.core.util.db.ConnectionHelper;
043import org.apache.jackrabbit.core.version.InternalVersionManagerImpl;
044import org.slf4j.LoggerFactory;
045
046/**
047 * DataStoreGarbageCollectorTask
048 */
049public class DataStoreGarbageCollectorTask extends AbstractMaintenanceTask implements MarkEventListener
050{
051    private static final int SYSTEM_GC_CALLS = 3;
052
053    /** The JackRabbit RepositoryImpl Context */
054    protected RepositoryContext _repositoryContext;
055
056    /** The JCR Session bound to this task. */
057    protected Session _session;
058
059    private GarbageCollector _garbageCollector;
060    private IterablePersistenceManager[] _pmList;
061
062    /** Internal counter for scanned node by the GC */
063    private int _scannedNodesCount;
064
065    @Override
066    protected void initialize() throws RepositoryException
067    {
068        // Create the repository and log in the session.
069        _repositoryContext = RepositoryContext.create(_repositoryConfig);
070        _session = _repositoryContext.getRepository().login(new SimpleCredentials("__MAINTENANCE_TASK__", "".toCharArray()));
071
072        // Create the GC Object
073        this._garbageCollector = _repositoryContext.getRepository().createDataStoreGarbageCollector();
074
075        // Workaround to get the list of the PersistenceManager
076        ArrayList<PersistenceManager> pmList = new ArrayList<>();
077        InternalVersionManagerImpl vm = _repositoryContext.getInternalVersionManager();
078        pmList.add(vm.getPersistenceManager());
079
080        String[] wspNames = _repositoryContext.getWorkspaceManager().getWorkspaceNames();
081        for (int i = 0; i < wspNames.length; i++)
082        {
083            pmList.add(getPM(wspNames[i]));
084        }
085
086        _pmList = new IterablePersistenceManager[pmList.size()];
087        for (int i = 0; i < pmList.size(); i++)
088        {
089            PersistenceManager pm = pmList.get(i);
090            if (!(pm instanceof IterablePersistenceManager))
091            {
092                _pmList = null;
093                break;
094            }
095            _pmList[i] = (IterablePersistenceManager) pm;
096        }
097
098        // Initialize the task progress object.
099        int count = 0; // number of item that will be scanned.
100
101        try
102        {
103            for (IterablePersistenceManager pm : _pmList)
104            {
105                count += pm.getAllNodeIds(null, 0).size();
106            }
107
108            // When scan is finished, progress is set at 70%.
109            float total = (10f / 7) * count;
110
111            _progress = new TaskProgress(Math.max(total, 1));
112        }
113        catch (Exception e)
114        {
115            _progress = new TaskProgress(0);
116            _progress.setInErrorState(e);
117            _logger.error(e.getLocalizedMessage(), e);
118        }
119    }
120    
121    @Override
122    protected void setLogger()
123    {
124        setLogger(LoggerFactory.getLogger(DataStoreGarbageCollectorTask.class));
125    }
126
127    @Override
128    protected void apply() throws RepositoryException
129    {
130        // call System.gc() a few times before running the data store garbage collection. 
131        // Please note System.gc() does not guarantee all objects are garbage collected.
132        for (int i = 0; i < SYSTEM_GC_CALLS; i++)
133        {
134            System.gc();
135        }
136
137        // Log some information about the ds.
138        long startSize = _reportDataStoreInfo(_garbageCollector.getDataStore());
139
140        // Sleep
141        if (_garbageCollector.getDataStore() instanceof FileDataStore)
142        {
143            // make sure the file is old (access time resolution is 2 seconds)
144            try
145            {
146                Thread.sleep(2000);
147            }
148            catch (InterruptedException e)
149            {
150                _logger.error(e.getLocalizedMessage(), e);
151                throw new RuntimeException(e); 
152            }
153        }
154
155        // GC Management
156        _garbageCollector.setMarkEventListener(this);
157        _garbageCollector.setPersistenceManagerScan(true);
158        // gc.setSleepBetweenNodes(0);
159
160        // Run GC
161        try
162        {
163            _scannedNodesCount = 0;
164            _logger.info("Scanning the repository nodes...");
165            _garbageCollector.mark();
166            _logger.info(_scannedNodesCount + " nodes scanned.");
167
168            _logger.info("Deleting unused items... Please be patient.");
169            int deleted = _garbageCollector.sweep();
170            _logger.info(deleted + " unused items deleted.");
171            if (_progress != null)
172            {
173                _progress.progressRelativePercentage(50);
174            }
175
176            _logger.info("Finalizing the process...");
177
178            // Compressing derby DATASTORE table.
179            if (_garbageCollector.getDataStore() instanceof DerbyDataStore)
180            {
181                _logger.info("Reclaiming unused space, this may take several minutes depending on the size of the data store.");
182                _logger.info("Please be patient.");
183
184                DbDataStore ds = (DbDataStore) _garbageCollector.getDataStore();
185                _derbyCompressTable(ds);
186            }
187        }
188        finally
189        {
190            _garbageCollector.close();
191        }
192
193        // Log some information about the ds.
194        long finalSize = _reportDataStoreInfo(_garbageCollector.getDataStore());
195        long freedSize = startSize - finalSize;
196        _logger.info("Size of cleared data : " + Math.round(freedSize / 1024) + "Ko");
197        _logger.info("The total released space on your disk can be different depending on the type of the data store used by your repository.");
198    }
199
200    @Override
201    protected void close()
202    {
203        if (_session != null)
204        {
205            _session.logout();
206        }
207
208        if (_repositoryContext != null && _repositoryContext.getRepository() != null)
209        {
210            _repositoryContext.getRepository().shutdown();
211        }
212
213        if (_progress != null)
214        {
215            _progress.progressRelativePercentage(100);
216        }
217    }
218
219    private long _reportDataStoreInfo(DataStore ds) throws DataStoreException
220    {
221        long count = 0;
222        long total = 0;
223        Iterator<DataIdentifier> it = ds.getAllIdentifiers();
224        while (it.hasNext())
225        {
226            count++;
227            DataIdentifier id = it.next();
228            total += ds.getRecord(id).getLength();
229        }
230
231        StringBuilder sb = new StringBuilder();
232        sb.append("Datastore item count : ").append(count).append(" ");
233        sb.append("[total size : ").append(Math.round(total / 1024)).append("Ko]");
234        _logger.info(sb.toString());
235
236        return total;
237    }
238
239    /**
240     * Reclaiming unused space. This is derby specific.
241     * By default, Derby does not return unused space to the operating system
242     * when updating or deleting data.
243     * @param ds The datastore
244     * @throws RepositoryException If an error occurs with the repository
245     */
246    private void _derbyCompressTable(DbDataStore ds) throws RepositoryException
247    {
248        DataSource dataSource = null;
249
250        try
251        {
252            ConnectionFactory cf = _repositoryConfig.getConnectionFactory();
253            if (ds.getDataSourceName() == null || "".equals(ds.getDataSourceName()))
254            {
255                dataSource = cf.getDataSource(ds.getDriver(), ds.getUrl(), ds.getUser(), ds.getPassword());
256            }
257            else
258            {
259                dataSource = cf.getDataSource(ds.getDataSourceName());
260            }
261        }
262        catch (SQLException e)
263        {
264            _logger.error(e.getLocalizedMessage(), e);
265            throw new RuntimeException(e);
266        }
267
268        if (dataSource != null)
269        {
270            ConnectionHelper conHelper = new ConnectionHelper(dataSource, false);
271            String sql = "CALL SYSCS_UTIL.SYSCS_COMPRESS_TABLE(CURRENT SCHEMA, ?, 0)";
272            String prefix = ds.getTablePrefix();
273            String schemaObjPrefix = ds.getSchemaObjectPrefix();
274            String table = prefix + schemaObjPrefix + "DATASTORE";
275
276            try
277            {
278                conHelper.query(sql, table);
279            }
280            catch (SQLException e)
281            {
282                _logger.error(e.getLocalizedMessage(), e);
283                throw new RuntimeException(e);
284            }
285        }
286        else
287        {
288            _logger.error("Unable to compress the Derby datastore, unused space has not been freed up.");
289        }
290
291    }
292
293    @Override
294    public void beforeScanning(Node n) throws RepositoryException
295    {
296        _scannedNodesCount++;
297        if (_progress != null)
298        {
299            _progress.progress();
300        }
301    }
302
303    /**
304     * Retrieves JackRabbit Persistence Manager for currently opened repository. This method uses
305     * Privileged access and will fail with security exception if used in environment with enabled security manager.
306     * @param workspaceName The workspace name
307     * @return Persistence manager used by repository.
308     * @throws RepositoryException If an error occurs while retrieving the persistence manager
309     */
310    protected PersistenceManager getPM(String workspaceName) throws RepositoryException
311    {
312        try
313        {
314            Object workspaceInfo = findAndInvokeMethod(_repositoryContext.getRepository(), "getWorkspaceInfo", new Object[] {workspaceName});
315            return (PersistenceManager) (findAndInvokeMethod(workspaceInfo, "getPersistenceManager", null));
316        }
317        catch (Exception e)
318        {
319            throw new RepositoryException(e);
320        }
321    }
322
323    private static Object findAndInvokeMethod(Object obj, String name, Object[] parameters) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException
324    {
325        Method m = null;
326        Method[] ms = obj.getClass().getDeclaredMethods();
327        for (int i = 0; i < ms.length; i++)
328        {
329            final Method x = ms[i];
330            if (x.getName().equals(name))
331            {
332                m = x;
333                m.setAccessible(true);
334                return m.invoke(obj, parameters);
335            }
336        }
337        
338        return null;
339    }
340}