dibbler/sqlalchemy/ext/serializer.py

"""Serializer/Deserializer objects for usage with SQLAlchemy query structures, 
allowing "contextual" deserialization.

Any SQLAlchemy query structure, either based on sqlalchemy.sql.*
or sqlalchemy.orm.* can be used.  The mappers, Tables, Columns, Session
etc. which are referenced by the structure are not persisted in serialized
form, but are instead re-associated with the query structure
when it is deserialized.

Usage is nearly the same as that of the standard Python pickle module::

    from sqlalchemy.ext.serializer import loads, dumps
    metadata = MetaData(bind=some_engine)
    Session = scoped_session(sessionmaker())
    
    # ... define mappers
    
    query = Session.query(MyClass).filter(MyClass.somedata=='foo').order_by(MyClass.sortkey)
    
    # pickle the query
    serialized = dumps(query)
    
    # unpickle.  Pass in metadata + scoped_session
    query2 = loads(serialized, metadata, Session)
    
    print query2.all()

Similar restrictions as when using raw pickle apply; mapped classes must be 
themselves be pickleable, meaning they are importable from a module-level
namespace.

The serializer module is only appropriate for query structures.  It is not
needed for:

* instances of user-defined classes.   These contain no references to engines, 
  sessions or expression constructs in the typical case and can be serialized directly.

* Table metadata that is to be loaded entirely from the serialized structure (i.e. is
  not already declared in the application).   Regular pickle.loads()/dumps() can 
  be used to fully dump any ``MetaData`` object, typically one which was reflected 
  from an existing database at some previous point in time.  The serializer module
  is specifically for the opposite case, where the Table metadata is already present
  in memory.

"""

from sqlalchemy.orm import class_mapper, Query
from sqlalchemy.orm.session import Session
from sqlalchemy.orm.mapper import Mapper
from sqlalchemy.orm.attributes import QueryableAttribute
from sqlalchemy import Table, Column
from sqlalchemy.engine import Engine
from sqlalchemy.util import pickle
import re
import base64
# Py3K
#from io import BytesIO as byte_buffer
# Py2K
from cStringIO import StringIO as byte_buffer
# end Py2K

# Py3K
#def b64encode(x):
#    return base64.b64encode(x).decode('ascii')
#def b64decode(x):
#    return base64.b64decode(x.encode('ascii'))
# Py2K
b64encode = base64.b64encode
b64decode = base64.b64decode
# end Py2K

__all__ = ['Serializer', 'Deserializer', 'dumps', 'loads']



def Serializer(*args, **kw):
    pickler = pickle.Pickler(*args, **kw)
        
    def persistent_id(obj):
        #print "serializing:", repr(obj)
        if isinstance(obj, QueryableAttribute):
            cls = obj.impl.class_
            key = obj.impl.key
            id = "attribute:" + key + ":" + b64encode(pickle.dumps(cls))
        elif isinstance(obj, Mapper) and not obj.non_primary:
            id = "mapper:" + b64encode(pickle.dumps(obj.class_))
        elif isinstance(obj, Table):
            id = "table:" + str(obj)
        elif isinstance(obj, Column) and isinstance(obj.table, Table):
            id = "column:" + str(obj.table) + ":" + obj.key
        elif isinstance(obj, Session):
            id = "session:"
        elif isinstance(obj, Engine):
            id = "engine:"
        else:
            return None
        return id
        
    pickler.persistent_id = persistent_id
    return pickler
    
our_ids = re.compile(r'(mapper|table|column|session|attribute|engine):(.*)')

def Deserializer(file, metadata=None, scoped_session=None, engine=None):
    unpickler = pickle.Unpickler(file)
    
    def get_engine():
        if engine:
            return engine
        elif scoped_session and scoped_session().bind:
            return scoped_session().bind
        elif metadata and metadata.bind:
            return metadata.bind
        else:
            return None
            
    def persistent_load(id):
        m = our_ids.match(id)
        if not m:
            return None
        else:
            type_, args = m.group(1, 2)
            if type_ == 'attribute':
                key, clsarg = args.split(":")
                cls = pickle.loads(b64decode(clsarg))
                return getattr(cls, key)
            elif type_ == "mapper":
                cls = pickle.loads(b64decode(args))
                return class_mapper(cls)
            elif type_ == "table":
                return metadata.tables[args]
            elif type_ == "column":
                table, colname = args.split(':')
                return metadata.tables[table].c[colname]
            elif type_ == "session":
                return scoped_session()
            elif type_ == "engine":
                return get_engine()
            else:
                raise Exception("Unknown token: %s" % type_)
    unpickler.persistent_load = persistent_load
    return unpickler

def dumps(obj, protocol=0):
    buf = byte_buffer()
    pickler = Serializer(buf, protocol)
    pickler.dump(obj)
    return buf.getvalue()
    
def loads(data, metadata=None, scoped_session=None, engine=None):
    buf = byte_buffer(data)
    unpickler = Deserializer(buf, metadata, scoped_session, engine)
    return unpickler.load()