Added cache.py
--
sim
--- /dev/null
+++ b/mp3togo/cache.py
@@ -0,0 +1,252 @@
+# - cache.py -
+# This file is part of mp3togo
+
+# Convert audio files to play on a mp3 player
+# Manage a cache of transformed files.
+#
+# (c) Simeon Veldstra 2006 <reallifesim@gmail.com>
+#
+# This software is free.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You may redistribute this program under the terms of the
+# GNU General Public Licence version 2
+# Available in this package or at http://www.fsf.org
+
+"""Cache files
+
+Pickle format: {checksum: (cachefname, hits, holds, since)}
+checksum: md5sum of original file
+cachefname: file name in the cache
+hits: number of cache hits and misses (total requests)
+holds: if > 0, file won't be deleted
+since: date cached
+
+"""
+
+import os
+import sys
+import threading
+import cPickle
+import md5
+import shutil
+import fcntl
+# Maybe lock the pickle with fcntl.fcntl?
+# The man page says flock won't work over NFS
+# We could have multiple processes accessing the
+# same cache over NFS.
+
+
+PICKLEPROTOCOL = 2
+
+
+class Cache:
+ """A cache for transformed files"""
+
+ def __init__(self, path, opts):
+ """May modify the options in opts"""
+ self.cuke = os.path.join(path, '2go.cache')
+ if not os.path.exists(self.cuke):
+ raise conf.ErrorNoCache
+ self.path = path
+ self._lock = threading.Lock()
+
+ try:
+ head = self._read_pickle()
+ except:
+ opts.log(1, "Error reading from file cache control pickle")
+ raise
+ # We might print a warning about changing these values
+ opts['cachesize'] = head.cachesize
+ opts['encoder'] = head.encoder
+ opts['encopts'] = head.encopts
+ opts['compfactor'] = head.compfactor
+ opts['brwarning'] = head.brwarning
+ opts['nonormal'] = head.nonormal
+ self._write_pickle(head)
+
+
+ def recover(self, hash, filename):
+ """Get a file from the cache.
+
+ Copy the file to filename and decrement holds"""
+ try:
+ self._lock.acquire()
+ head = self._read_pickle()
+ d = head.d
+ shutil.copyfile(d[hash][0], filename)
+ d[hash] = (d[hash][0], d[hash][1], d[hash][2] - 1)
+ self._write_pickle(head)
+ self._lock.release()
+ except:
+ return False
+ return True
+
+ def search(self, filename, justlook=False):
+ """Find a file in the cache.
+
+ filename is the name of the uncoverted file
+ Returns the hash and increments hits and holds if the file exists,
+ if justlook is true, holds is not touched,
+ returns None if the file is not found.
+ If the file is not found, its hash is added to
+ the pickle with an empty filename and hits set to 1"""
+ self._lock.acquire()
+ head = self._read_pickle()
+ sum = checksum(filename)
+ hold = int(not bool(justlook))
+ d = head.d
+ if sum in d.keys():
+ if d[sum][0]:
+ d[sum] = (d[sum][0], d[sum][1] + 1, d[sum][2] + hold)
+ ret = sum
+ else:
+ d[sum] = ('', d[sum][1] + 1, 0)
+ ret = False
+ else:
+ d[sum] = ('', 1, 0)
+ ret = False
+ self._write_pickle(head)
+ self._lock.release()
+ return ret
+
+ def stash(self, filename, sum):
+ """Add a file to the cache, if it fits
+
+ sum is the hash of the unconverted file
+ filename is the name of the converted file"""
+ self._lock.acquire()
+ head = self._read_pickle()
+ d = head.d
+ if d.has_key(sum) and d[sum][0]:
+ # It's already there
+ self._write_pickle(head)
+ self._lock.release()
+ return True
+
+ fits = False
+ free = head.cachesize - head.bytes
+ # Should use blocks * blocksize to get accurate space requirement
+ # unfortunately, the result of st_blksize * st_blocks makes no sense.
+ size = os.stat(filename).st_size
+ if size < free:
+ fits = True
+ else:
+ if d.has_key(sum):
+ rank = d[sum][1]
+ else:
+ rank = 0
+ # (Hits, checksum) if holds == 0
+ sl = [[x[1][1], x[0]] for x in d.items() if x[1][2] == 0 and x[1][0]]
+ sl.sort()
+ # Delete lower ranking files until it either fits or is outranked
+ while not fits and sl and sl[0][0] < rank:
+ lsum = sl[0][1]
+ lname = d[lsum][0]
+ lsize = os.stat(lname).st_size
+ os.unlink(lname)
+ d[lsum] = ('', d[lsum][1], 0)
+ head.bytes -= lsize
+ del sl[0]
+ if size < (head.cachesize - head.bytes):
+ fits = True
+
+ if fits:
+ cachename = os.path.basename(filename)
+ cachename = os.path.join(self.path, cachename)
+ shutil.copyfile(filename, cachename)
+ if d.has_key(sum):
+ d[sum] = (cachename, d[sum][1], 0)
+ else:
+ d[sum] = (cachename, 0, 0)
+ head.bytes += size
+
+ self._write_pickle(head)
+ self._lock.release()
+ return fits
+
+ def file(self, hash):
+ head = self._read_pickle()
+ name = head.d[hash][0]
+ self._write_pickle()
+ return name
+
+ def release(self, hash):
+ """If you change your mind about wanting the file call release"""
+ head = self._read_pickle()
+ d = head.d
+ holds = d[hash][2] - 1
+ if holds < 0:
+ holds = 0
+ d[hash] = (d[hash][0], d[hash][1], holds)
+ self._write_pickle()
+
+
+ def _read_pickle(self):
+ # Open the file for reading and grab an advisory lock
+ # possibly blocking. Read in the pickle and keep the lock.
+ self._fp = file(self.cuke, 'r')
+ fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX)
+ head = cPickle.load(self._fp)
+ return head
+
+ def _write_pickle(self, head=None):
+ # Ignore the advisory lock and open the file to write out
+ # the pickle. Drop the lock acquired by _read_pickle and
+ # close the file object it opened.
+ if head:
+ fp = file(self.cuke, 'w')
+ cPickle.dump(head, fp, PICKLEPROTOCOL)
+ fp.close()
+ fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
+ self._fp.close()
+
+
+def create(path, opts):
+ """Create a new file cache"""
+ try:
+ if not os.path.exists(path):
+ os.makedirs(path)
+ cuke = os.path.join(path, '2go.cache')
+ if os.path.exists(cuke):
+ opts.log(1, "Cache exists, remove first")
+ raise Exception
+ data = CacheHead(opts)
+ fp = file(cuke, 'w')
+ cPickle.dump(data, fp, PICKLEPROTOCOL)
+ fp.close()
+ return True
+ except:
+ opts.log(1, "Error Creating Cache control pickle")
+ return False
+
+
+class CacheHead:
+ """Data structure to manage a cache.
+
+ Pickled and left at the top level of a cache in 2go.cache
+ """
+
+ def __init__(self, opts):
+ """Create a new cache"""
+ self.d = {}
+ self.bytes = 0
+ self.cachesize = opts['cachesize']
+ self.encoder = opts['encoder']
+ self.encopts = opts['encopts']
+ self.compfactor = opts['compfactor']
+ self.brwarning = opts['brwarning']
+ self.nonormal = opts['nonormal']
+
+
+def checksum(filename):
+ f = file(filename, 'r')
+ sum = md5.new(f.read()).hexdigest()
+ f.close()
+ return sum
+
+
projects / mp3togo / commitdiff