001 /*
002 @license.text@
003 */
004 package biz.hammurapi.cache;
005
006 import java.io.ByteArrayOutputStream;
007 import java.io.File;
008 import java.io.FileInputStream;
009 import java.io.FileOutputStream;
010 import java.io.IOException;
011 import java.io.InputStreamReader;
012 import java.io.ObjectInputStream;
013 import java.io.ObjectOutputStream;
014 import java.io.Serializable;
015 import java.sql.SQLException;
016 import java.util.ArrayList;
017 import java.util.HashSet;
018 import java.util.Iterator;
019 import java.util.List;
020 import java.util.Set;
021 import java.util.Timer;
022 import java.util.TimerTask;
023
024 import biz.hammurapi.CarryOverException;
025 import biz.hammurapi.RuntimeException;
026 import biz.hammurapi.cache.AbstractProducer;
027 import biz.hammurapi.cache.Cache;
028 import biz.hammurapi.cache.Entry;
029 import biz.hammurapi.cache.Producer;
030 import biz.hammurapi.cache.sql.CacheEntry;
031 import biz.hammurapi.cache.sql.FileCacheEngine;
032 import biz.hammurapi.config.ConfigurationException;
033 import biz.hammurapi.sql.DataIterator;
034 import biz.hammurapi.sql.SQLProcessor;
035 import biz.hammurapi.sql.SQLRuntimeException;
036 import biz.hammurapi.sql.Transaction;
037 import biz.hammurapi.sql.hypersonic.HypersonicStandaloneDataSource;
038 import biz.hammurapi.util.Acceptor;
039
040
041 /**
042 * Caches objects in files.
043 * @author Pavel Vlasov
044 * @version $Revision: 1.7 $
045 */
046 public class FileCache extends AbstractProducer implements Cache {
047
048 private static final int CLEANUP_INTERVAL = 10*60*1000;
049 private static final int MAX_FILES_PER_DIRECTORY = 10000;
050 private static final int ACTIVE_DIRS = 5;
051 private HypersonicStandaloneDataSource ds;
052 private FileCacheEngine engine;
053 private File dataDir;
054 private long size;
055 private long maxSize;
056 private Producer producer;
057
058 private class DirectoryInfo {
059 File directory;
060 int fileCount;
061
062 DirectoryInfo() {
063 synchronized (FileCache.this) {
064 do {
065 directory=new File(dataDir, Long.toString(System.currentTimeMillis(), Character.MAX_RADIX));
066 } while (directory.exists());
067
068 directory.mkdir();
069 fileCount=directory.listFiles().length;
070 }
071 }
072
073 DirectoryInfo(File dir) {
074 directory=dir;
075 fileCount=directory.listFiles().length;
076 }
077
078 File nextFile() throws IOException {
079 fileCount++;
080 return File.createTempFile("cache_", ".tmp", directory);
081 }
082 }
083
084 private List directories=new ArrayList();
085
086 private File nextFile() throws IOException {
087 Iterator it=directories.iterator();
088 while (it.hasNext()) {
089 DirectoryInfo di=(DirectoryInfo) it.next();
090 if (di.fileCount>MAX_FILES_PER_DIRECTORY) {
091 it.remove();
092 }
093 }
094
095 while (directories.size()<ACTIVE_DIRS) {
096 directories.add(new DirectoryInfo());
097 }
098
099 return ((DirectoryInfo) directories.get(((int) (Math.random() * Integer.MAX_VALUE)) % ACTIVE_DIRS)).nextFile();
100 }
101
102 /**
103 *
104 * @param producer
105 * @param dir
106 * @param maxSize - Maximum cache size. Number <=0 means no limit
107 * @throws IOException
108 */
109 public FileCache(Producer producer, File dir, long maxSize) throws IOException {
110 super();
111 this.maxSize=maxSize>0 ? maxSize : Long.MAX_VALUE;
112 this.producer=producer;
113 if (producer!=null) {
114 producer.addCache(this);
115 }
116
117 dataDir=new File(dir, "data");
118 if (!dataDir.exists()) {
119 dataDir.mkdir();
120 }
121 if (!dataDir.exists()) {
122 throw new IOException("Cannot create directory "+dataDir.getAbsolutePath());
123 }
124 if (!dataDir.isDirectory()) {
125 throw new IOException("Not a directory: "+dataDir.getAbsolutePath());
126 }
127
128 File[] dirs=dataDir.listFiles();
129 for (int i=0; i<dirs.length; i++) {
130 directories.add(new DirectoryInfo(dirs[i]));
131 File[] entries=dirs[i].listFiles();
132 for (int j=0; j<entries.length; i++) {
133 size+=entries[i].length();
134 }
135 }
136
137 try {
138 ds=new HypersonicStandaloneDataSource(
139 new File(dir, "entries").getAbsolutePath(),
140 new Transaction() {
141
142 public boolean execute(SQLProcessor processor) throws SQLException {
143 try {
144 processor.executeScript(new InputStreamReader(getClass().getResourceAsStream("FileCache.sql")));
145 } catch (IOException e) {
146 throw new CarryOverException(e);
147 }
148 return true;
149 }
150 });
151
152 engine = new FileCacheEngine(new SQLProcessor(ds, null));
153 } catch (ClassNotFoundException e) {
154 throw new IOException("Caused by "+e);
155 } catch (SQLException e) {
156 throw new IOException("Caused by "+e);
157 } catch (CarryOverException e) {
158 throw (IOException) e.getCause();
159 }
160 }
161
162 private boolean shutDown=false;
163
164 private void checkShutdown() {
165 if (shutDown) {
166 throw new IllegalStateException("Shut down");
167 }
168 }
169
170 /**
171 * Shuts down entries database and janitor thread.
172 */
173 public void stop() {
174 shutDown=true;
175 ds.shutdown();
176 janitorTask.cancel();
177 if (isOwnTimer) {
178 timer.cancel();
179 }
180 }
181
182 synchronized public void put(Object key, Object value, long time, long expirationTime) {
183 checkShutdown();
184 if (key instanceof String && value instanceof Serializable) {
185 try {
186 remove(key);
187
188 // TODO - modify to avoid running into OS limitations of files per dir.
189 File out=nextFile();
190 ObjectOutputStream oos=new ObjectOutputStream(new FileOutputStream(out));
191 try {
192 oos.writeObject(value);
193 } finally {
194 oos.close();
195 }
196
197 String fileName = out.getParentFile().getName()+File.separator+out.getName();
198 //System.out.println(key + " -> " + fileName);
199 engine.insertCacheEntry((String) key, fileName, time, expirationTime, 0);
200 size+=out.length();
201
202 if (size>maxSize) {
203 Iterator it=engine.getCacheEntryLastAccessOrdered().iterator();
204 while (size>maxSize && it.hasNext()) {
205 CacheEntry entry=(CacheEntry) it.next();
206 File file = new File(dataDir, entry.getValueFile());
207 long length=file.length();
208 if (file.delete()) {
209 size-=length;
210 }
211 engine.deleteCacheEntry(entry.getEntryKey());
212 }
213
214 ((DataIterator) it).close();
215 }
216 } catch (IOException e) {
217 e.printStackTrace();
218 // ignore
219 } catch (SQLException e) {
220 e.printStackTrace();
221 // ignore
222 }
223 } else {
224 System.err.println("WARN: Cannot serialize "+value.getClass());
225 }
226 }
227
228 private TimerTask janitorTask = new TimerTask() {
229
230 public void run() {
231 try {
232 long now = System.currentTimeMillis();
233 Iterator it=engine.getCacheEntryExpiresLE(now).iterator();
234 synchronized (FileCache.this) {
235 while (it.hasNext()) {
236 File file = new File(dataDir, ((CacheEntry) it.next()).getValueFile());
237 long length=file.length();
238 if (file.delete()) {
239 size-=length;
240 }
241 }
242 }
243 engine.deleteCacheEntryExpiresLE(now);
244 } catch (SQLException e) {
245 e.printStackTrace();
246 // ignore
247 } catch (SQLRuntimeException e) {
248 e.printStackTrace();
249 // ignore
250 }
251 }
252 };
253
254 synchronized public void clear() {
255 checkShutdown();
256 try {
257 engine.deleteCacheEntry();
258 } catch (SQLException e) {
259 e.printStackTrace();
260 // ignore
261 }
262 File[] entries=dataDir.listFiles();
263 for (int i=0; i<entries.length; i++) {
264 long length=entries[i].length();
265 if (entries[i].delete()) {
266 size-=length;
267 }
268 }
269 }
270
271 synchronized public void remove(Object key) {
272 checkShutdown();
273 if (key instanceof String) {
274 try {
275 CacheEntry cacheEntry = engine.getCacheEntry((String) key);
276 if (cacheEntry!=null) {
277 File file = new File(dataDir, cacheEntry.getValueFile());
278 long length=file.length();
279 if (file.delete()) {
280 size-=length;
281 }
282 engine.deleteCacheEntry((String) key);
283 }
284 } catch (SQLException e) {
285 e.printStackTrace();
286 }
287 }
288 onRemove(key);
289 }
290
291 public Entry get(Object key) {
292 checkShutdown();
293 Entry ret=key instanceof String ? _get((String) key) : null;
294 if (ret==null && producer!=null) {
295 ret=producer.get(key);
296 if (ret!=null) {
297 put(key, ret.get(), ret.getTime(), ret.getExpirationTime());
298 }
299 }
300 //System.out.println("*** "+key+" Restored ---> "+ret);
301 return ret;
302 }
303
304 synchronized private Entry _get(String key) {
305 try {
306 CacheEntry entry=engine.getCacheEntry(key);
307 if (entry==null) {
308 return null;
309 }
310 long now=System.currentTimeMillis();
311 if (entry.getEntryExpires()<=0 || now<=entry.getEntryExpires()) {
312 engine.setLastAccess(now, key);
313 File valueFile=new File(dataDir, entry.getValueFile());
314 //System.out.println("Restoring "+key+" <- "+entry.getValueFile()+" ("+valueFile.length()+" bytes)");
315 ObjectInputStream ois=new ObjectInputStream(new FileInputStream(valueFile));
316 try {
317 final Object value=ois.readObject();
318 final long expires=entry.getEntryExpires();
319 final long time=entry.getEntryTime();
320 return new Entry() {
321
322 public long getExpirationTime() {
323 return expires;
324 }
325
326 public long getTime() {
327 return time;
328 }
329
330 public Object get() {
331 return value;
332 }
333 };
334 } finally {
335 ois.close();
336 }
337 }
338 return null;
339 } catch (IOException e) {
340 e.printStackTrace();
341 remove(key);
342 return null;
343 } catch (ClassNotFoundException e) {
344 e.printStackTrace();
345 remove(key);
346 return null;
347 } catch (SQLException e) {
348 e.printStackTrace();
349 return null;
350 }
351 }
352
353 public static void main(String[] args) throws Exception {
354 FileCache fileCache=new FileCache(new Producer() {
355
356 public Entry get(Object key) {
357 try {
358 int size=(int) (Math.random()*20000);
359 ByteArrayOutputStream bos=new ByteArrayOutputStream();
360 ObjectOutputStream oos=new ObjectOutputStream(bos);
361
362 while (bos.size()<size) {
363 oos.writeObject(key+"_at_"+bos.size());
364 }
365 oos.close();
366 bos.close();
367
368 final byte[] bytes = bos.toByteArray();
369 return new Entry() {
370
371 public long getExpirationTime() {
372 return 0;
373 }
374
375 public long getTime() {
376 return 0;
377 }
378
379 public Object get() {
380 return bytes;
381 }
382
383 };
384 } catch (IOException e) {
385 e.printStackTrace();
386 return null;
387 }
388 }
389
390 public void addCache(Cache cache) {
391 // No action
392 }
393
394 public Set keySet() {
395 return null;
396 }
397
398 },
399 new File("C:\\_fileCacheTest"),
400 100000);
401
402 for (int i=0; i<300; i++) {
403 fileCache.get("Key_"+i);
404 }
405
406 fileCache.stop();
407 }
408
409 public void remove(Acceptor acceptor) {
410 checkShutdown();
411 Iterator it=engine.getCacheEntry().iterator();
412 try {
413 CacheEntry ce=(CacheEntry) it.next();
414 if (acceptor.accept(ce.getEntryKey())) {
415 try {
416 File file = new File(dataDir, engine.getCacheEntry(ce.getEntryKey()).getValueFile());
417 long length=file.length();
418 if (file.delete()) {
419 size-=length;
420 }
421 engine.deleteCacheEntry(ce.getEntryKey());
422 } catch (SQLException e) {
423 e.printStackTrace();
424 }
425 }
426 onRemove(ce.getEntryKey());
427 } finally {
428 try {
429 ((DataIterator) it).close();
430 } catch (SQLException e) {
431 throw new RuntimeException(e);
432 }
433 }
434 }
435
436 public Set keySet() {
437 HashSet ret = new HashSet();
438 Iterator it=engine.getCacheEntry().iterator();
439 while (it.hasNext()) {
440 ret.add(((CacheEntry) it.next()).getEntryKey());
441 }
442
443 if (producer!=null) {
444 Set pkeys=producer.keySet();
445 if (pkeys!=null) {
446 ret.addAll(pkeys);
447 }
448 }
449
450 return ret;
451 }
452
453 public boolean isActive() {
454 return !shutDown;
455 }
456
457 private Timer timer;
458 private boolean isOwnTimer;
459
460 public void start() throws ConfigurationException {
461 if (timer==null) {
462 timer=new Timer();
463 isOwnTimer=true;
464 }
465
466 timer.schedule(janitorTask, CLEANUP_INTERVAL, CLEANUP_INTERVAL);
467 }
468
469 public void setOwner(Object owner) {
470 // Nothing
471
472 }
473
474 }