u = unicode(encoded_string[,encoding=<default encoding>][,errors="strict"])

u = u'<unicode-escape encoded Python string>'

u = ur'<raw-unicode-escape encoded Python string>'

u'abc'          -> U+0061 U+0062 U+0063
u'\u1234'       -> U+1234
u'abc\u1234\n'  -> U+0061 U+0062 U+0063 U+1234 U+005c

print u := print u.encode()   # using the <default encoding>

str(u)  := u.encode()         # using the <default encoding>

repr(u) := "u%s" % repr(u.encode('unicode-escape'))

ord(u[:1]) (this is the standard ord() extended to work with Unicode
            objects)
  --> Unicode ordinal number (32-bit)

unichr(i)
    --> Unicode object for character i (provided it is 32-bit);
        ValueError otherwise

u + s := u + unicode(s)

s + u := unicode(s) + u

string.join((s,u),sep) := (s + sep) + u

sep.join((s,u)) := (s + sep) + u

codecs.register(search_function)

codecs.lookup(encoding)

'utf-8':              8-bit variable length encoding
'utf-16':             16-bit variable length encoding (little/big endian)
'utf-16-le':          utf-16 but explicitly little endian
'utf-16-be':          utf-16 but explicitly big endian
'ascii':              7-bit ASCII codepage
'iso-8859-1':         ISO 8859-1 (Latin 1) codepage
'unicode-escape':     See Unicode Constructors for a definition
'raw-unicode-escape': See Unicode Constructors for a definition
'native':             Dump of the Internal Format used by Python

class Codec:

    """Defines the interface for stateless encoders/decoders.

       The .encode()/.decode() methods may implement different
       error handling schemes by providing the errors argument.
       These string values are defined:

         'strict'  - raise an error (or a subclass)
         'ignore'  - ignore the character and continue with the next
         'replace' - replace with a suitable replacement character;
                     Python will use the official U+FFFD
                     REPLACEMENT CHARACTER for the builtin Unicode
                     codecs.
    """

    def encode(self,input,errors='strict'):

        """Encodes the object input and returns a tuple (output
           object, length consumed).

           errors defines the error handling to apply.  It
           defaults to 'strict' handling.

           The method may not store state in the Codec instance.
           Use StreamCodec for codecs which have to keep state in
           order to make encoding/decoding efficient.
        """

    def decode(self,input,errors='strict'):

        """Decodes the object input and returns a tuple (output
           object, length consumed).

           input must be an object which provides the
           bf_getreadbuf buffer slot.  Python strings, buffer
           objects and memory mapped files are examples of objects
           providing this slot.

           errors defines the error handling to apply.  It
           defaults to 'strict' handling.

           The method may not store state in the Codec instance.
           Use StreamCodec for codecs which have to keep state in
           order to make encoding/decoding efficient.

        """

class StreamWriter(Codec):

    def __init__(self,stream,errors='strict'):

        """Creates a StreamWriter instance.

           stream must be a file-like object open for writing
           (binary) data.

           The StreamWriter may implement different error handling
           schemes by providing the errors keyword argument.
           These parameters are defined:

             'strict' - raise a ValueError (or a subclass)
             'ignore' - ignore the character and continue with the next
             'replace'- replace with a suitable replacement character
        """
        self.stream = stream
        self.errors = errors

    def write(self,object):

        """Writes the object's contents encoded to self.stream.
        """
        data, consumed = self.encode(object,self.errors)
        self.stream.write(data)

    def writelines(self, list):

        """Writes the concatenated list of strings to the stream
           using .write().
        """
        self.write(''.join(list))

    def reset(self):

        """Flushes and resets the codec buffers used for keeping state.

           Calling this method should ensure that the data on the
           output is put into a clean state, that allows appending
           of new fresh data without having to rescan the whole
           stream to recover state.
        """
        pass

    def __getattr__(self,name, getattr=getattr):

        """Inherit all other methods from the underlying stream.
        """
        return getattr(self.stream,name)


class StreamReader(Codec):

    def __init__(self,stream,errors='strict'):

        """Creates a StreamReader instance.

           stream must be a file-like object open for reading
           (binary) data.

           The StreamReader may implement different error handling
           schemes by providing the errors keyword argument.
           These parameters are defined:

             'strict' - raise a ValueError (or a subclass)
             'ignore' - ignore the character and continue with the next
             'replace'- replace with a suitable replacement character;
        """
        self.stream = stream
        self.errors = errors

    def read(self,size=-1):

        """Decodes data from the stream self.stream and returns the
           resulting object.

           size indicates the approximate maximum number of bytes
           to read from the stream for decoding purposes.  The
           decoder can modify this setting as appropriate.  The
           default value -1 indicates to read and decode as much
           as possible.  size is intended to prevent having to
           decode huge files in one step.

           The method should use a greedy read strategy meaning
           that it should read as much data as is allowed within
           the definition of the encoding and the given size, e.g.
           if optional encoding endings or state markers are
           available on the stream, these should be read too.
        """
        # Unsliced reading:
        if size < 0:
            return self.decode(self.stream.read())[0]

        # Sliced reading:
        read = self.stream.read
        decode = self.decode
        data = read(size)
        i = 0
        while 1:
            try:
                object, decodedbytes = decode(data)
            except ValueError,why:
                # This method is slow but should work under pretty
                # much all conditions; at most 10 tries are made
                i = i + 1
                newdata = read(1)
                if not newdata or i > 10:
                    raise
                data = data + newdata
            else:
                return object

    def readline(self, size=None):

        """Read one line from the input stream and return the
           decoded data.

           Note: Unlike the .readlines() method, this method
           inherits the line breaking knowledge from the
           underlying stream's .readline() method -- there is
           currently no support for line breaking using the codec
           decoder due to lack of line buffering.  Subclasses
           should however, if possible, try to implement this
           method using their own knowledge of line breaking.

           size, if given, is passed as size argument to the
           stream's .readline() method.
        """
        if size is None:
            line = self.stream.readline()
        else:
            line = self.stream.readline(size)
        return self.decode(line)[0]

    def readlines(self, sizehint=0):

        """Read all lines available on the input stream
           and return them as list of lines.

           Line breaks are implemented using the codec's decoder
           method and are included in the list entries.

           sizehint, if given, is passed as size argument to the
           stream's .read() method.
        """
        if sizehint is None:
            data = self.stream.read()
        else:
            data = self.stream.read(sizehint)
        return self.decode(data)[0].splitlines(1)

    def reset(self):

        """Resets the codec buffers used for keeping state.

           Note that no stream repositioning should take place.
           This method is primarily intended to be able to recover
           from decoding errors.

        """
        pass

    def __getattr__(self,name, getattr=getattr):

        """ Inherit all other methods from the underlying stream.
        """
        return getattr(self.stream,name)

BOM_BE: '\376\377'
  (corresponds to Unicode U+0000FEFF in UTF-16 on big endian
   platforms == ZERO WIDTH NO-BREAK SPACE)

BOM_LE: '\377\376'
  (corresponds to Unicode U+0000FFFE in UTF-16 on little endian
   platforms == defined as being an illegal Unicode character)

BOM4_BE: '\000\000\376\377'
  (corresponds to Unicode U+0000FEFF in UCS-4)

BOM4_LE: '\377\376\000\000'
  (corresponds to Unicode U+0000FFFE in UCS-4)

'%s': For Unicode objects this will cause coercion of the
      whole format string to Unicode.  Note that you should use
      a Unicode format string to start with for performance
      reasons.

'%s': Python strings are interpreted as Unicode
      string using the <default encoding>.  Unicode objects are
      taken as is.

u"%s %s" % (u"abc", "abc")  ==  u"abc abc"

static PyObject *
test_parser(PyObject *self,
            PyObject *args)
{
    PyObject *str;
    const char *encoding = "latin-1";
    char *buffer = NULL;
    int buffer_len = 0;

    if (!PyArg_ParseTuple(args, "es#:test_parser",
                          encoding, &buffer, &buffer_len))
        return NULL;
    if (!buffer) {
        PyErr_SetString(PyExc_SystemError,
                        "buffer is NULL");
        return NULL;
    }
    str = PyString_FromStringAndSize(buffer, buffer_len);
    PyMem_Free(buffer);
    return str;
}

static PyObject *
test_parser(PyObject *self,
            PyObject *args)
{
    PyObject *str;
    const char *encoding = "latin-1";
    char *buffer = NULL;

    if (!PyArg_ParseTuple(args, "es:test_parser",
                          encoding, &buffer))
        return NULL;
    if (!buffer) {
        PyErr_SetString(PyExc_SystemError,
                        "buffer is NULL");
        return NULL;
    }
    str = PyString_FromString(buffer);
    PyMem_Free(buffer);
    return str;
}

static PyObject *
test_parser(PyObject *self,
            PyObject *args)
{
    PyObject *str;
    const char *encoding = "latin-1";
    char _buffer[10];
    char *buffer = _buffer;
    int buffer_len = sizeof(_buffer);

    if (!PyArg_ParseTuple(args, "es#:test_parser",
                          encoding, &buffer, &buffer_len))
        return NULL;
    if (!buffer) {
        PyErr_SetString(PyExc_SystemError,
                        "buffer is NULL");
        return NULL;
    }
    str = PyString_FromStringAndSize(buffer, buffer_len);
    return str;
}

.encode([encoding=<default encoding>][,errors="strict"])
   --> see Unicode Output

.splitlines([include_breaks=0])
   --> breaks the Unicode string into a list of (Unicode) lines;
       returns the lines with line breaks included, if
       include_breaks is true.  See Line Breaks for a
       specification of how line breaking is done.