arrayman.py

# Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.dev/sumo
# Copyright (C) 2016-2025 German Aerospace Center (DLR) and others.
# SUMOPy module
# Copyright (C) 2012-2021 University of Bologna - DICAM
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License 2.0 which is available at
# https://www.eclipse.org/legal/epl-2.0/
# This Source Code may also be made available under the following Secondary
# Licenses when the conditions for such availability set forth in the Eclipse
# Public License 2.0 are satisfied: GNU General Public License, version 2
# or later which is available at
# https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html
# SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-or-later

# @file    arrayman.py
# @author  Joerg Schweizer
# @date    2012

from classman import *
import numpy as np


class ArrayConfMixin:
    def __init__(self, attrname, default, dtype=None, is_index=False, **attrs):
        self._is_index = is_index
        self._dtype = dtype
        AttrConf.__init__(self,  attrname, default,
                          struct='array',
                          **attrs)

        if is_index:
            self._init_indexing()

    def get_dtype(self):
        return self._dtype

    def convert_type(self, array):
        return np.array(array, dtype=self._dtype)

    def get_defaults(self, ids):
        # create a list, should work for all types and dimensions
        # default can be scalar or an array of any dimension
        # print '\n\nget_defaults',self._name,ids
        default = self.get_default()

        if hasattr(default, '__iter__'):
            default = np.asarray(default)
            if self._dtype is not None:
                dtype = self._dtype
            else:
                dtype = type(default.flatten()[0])
            # print ' default=',default,len(default)
            if len(ids) > 0:
                defaults = np.array(len(ids)*[default], dtype)
                # print '  size,type',len(ids)*[default], type(default.flatten()[0])
            else:
                #defaults = np.zeros(  (0,len(default)),type(default.flatten()[0]) )
                defaults = np.zeros((0,)+default.shape, dtype)
            # print '  return',defaults,defaults.shape,defaults.dtype
            return defaults
        else:
            if self._dtype is not None:
                dtype = self._dtype
            else:
                dtype = type(default)
            #defaults=  np.array(  len(ids)*[default], dtype )
            # print '  return 1D',defaults,defaults.shape,defaults.dtype
            return np.array(len(ids)*[default], dtype)

    def get_init(self):
        """
        Returns initialization of attribute.
        Usually same as get_default for scalars.
        Overridden by table configuration classes
        """
        ids = self._manager.get_ids()

        # print '\n\nget_init',self.attrname,ids,self._is_localvalue
        values = self.get_defaults(ids)

        # store locally if required
        if self._is_localvalue:
            self.value = values
        # pass on to calling instance
        # in this cas the data is stored under self._obj
        return values

    def get_ids_sorted(self):
        inds = self._manager.get_inds()
        return self._manager.get_ids(inds[np.argsort(self.get_value()[inds])])

        #sortarray = np.concatenate((self.get_value()[inds],inds))

    def _del_rows(self, ids_del, inds_remain):
        """
        Multiple row delete method used by attrsman
        """
        self.set_value(self.get_value()[inds_remain])

    def delete_ind(self, i):
        # called from del_rows
        if self._is_index:
            _id = self._manager._ids[i]
            self.del_index(_id)
        arr = self.get_value()
        self.set_value(np.concatenate((arr[:i], arr[i+1:])))

    def __delitem__(self, ids):
        # print '        before=\n',self.__dict__[attr]
        #attr = self.attrconf.get_attr()
        if hasattr(ids, '__iter__'):
            for i in self._manager._inds[ids]:
                self.delete_ind[i]
        else:
            self.delete_ind(self._manager._inds[ids])

    def __getitem__(self, ids):
        # print '__getitem__',key
        return self.get_value()[self._manager._inds[ids]]

    def __setitem__(self, ids, values):
        # print '__setitem__',ids,values,type(self.get_value()),self.get_value().dtype

        if self._is_index:
            if hasattr(ids, '__iter__'):
                self.set_indices(ids, values)

            else:
                self.set_index(ids, values)

        self.get_value()[self._manager._inds[ids]] = values

    def set(self, ids, values):
        if values is None:
            return

        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            _values = np.array([values], self._dtype)

        else:
            _ids = ids
            _values = np.array(values, self._dtype)
        # print 'set',  _ids ,_values
        self[_ids] = _values
        if self._is_index:
            self.set_indices(_ids, _values)
        self._is_modified = True

    def set_plugin(self, ids, values):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            _values = np.array([values], self._dtype)

        else:
            _ids = ids
            _values = np.array(values, self._dtype)

        self[ids] = _values
        if self._is_index:
            self.set_indices(_ids, _values)
        self._is_modified = True
        self.plugin.exec_events_ids(EVTSETITEM, _ids)

    def add(self, ids, values=None):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.array([values], self._dtype)
            else:
                _values = self.get_defaults(_ids)

        else:
            # if values is None:
            #    print 'WARNING:',self.attrname, ids,self._dtype
            _ids = ids
            if values is not None:
                _values = np.array(values, self._dtype)
            else:
                _values = self.get_defaults(_ids)
        # if values  is None:
        #    _values = self.get_defaults(_ids)

        # print 'add ids',self.attrname,ids,_ids,self._is_modified
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        #newvalue = np.concatenate((self.get_value(),_values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))
        # print '  done:',self.attrname,self.get_value()
        if self._is_index:
            self.add_indices(_ids, _values)
        self._is_modified = True

    def add_plugin(self, ids, values=None):
        # print 'add_plugin',self.attrname
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.array([values], self._dtype)

        else:
            _ids = ids
            if values is not None:
                _values = np.array(values, self._dtype)

        if values is None:
            _values = self.get_defaults(_ids)
        self._is_modified = True
        # print 'add ids',self.attrname,ids,_ids,self._is_modified
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        #newvalue = np.concatenate((self.get_value(),_values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))
        # print '  done:',self.attrname,self.get_value()

        if self._is_index:
            self.add_indices(_ids, _values)

        self.plugin.exec_events_ids(EVTADDITEM, _ids)

    # use original one from AttrConfig
    # def _write_xml_value(self,val,fd):
    #    #print 'write_xml',self.xmltag,type(val),hasattr(val, '__iter__')
    #    if hasattr(val, '__iter__'):
    #        if len(val)>0:
    #            if hasattr(val[0], '__iter__'):
    #                # matrix
    #                fd.write(xm.mat(self.xmltag,val))
    #            else:
    #                if type(val)==np.ndarray:
    #                    # vector
    #                    fd.write(xm.arr(self.xmltag,val,sep=','))
    #                else:
    #                    # list
    #                    fd.write(xm.arr(self.xmltag,val))
    #        else:
    #            # empty list
    #            fd.write(xm.arr(self.xmltag,val))
    #    else:
    #        # scalar number or string
    #        fd.write(xm.num(self.xmltag,val))

    def format_value(self, _id, show_unit=False, show_parentesis=False):
        # TODO: handle also linked ids, etc...
        if show_unit:
            unit = ' '+self.format_unit(show_parentesis)
        else:
            unit = ''
        # return repr(self[_id])+unit

        #self.min = minval
        #self.max = maxval
        #self.digits_integer = digits_integer
        #self.digits_fraction = digits_fraction
        val = self[_id]
        tt = type(val)

        if tt in (np.int, np.int32, np.float64):
            return str(val)+unit

        elif tt in (np.float, np.float32, np.float64):
            if hasattr(self, 'digits_fraction'):
                digits_fraction = self.digits_fraction
            else:
                digits_fraction = 3

            s = "%."+str(digits_fraction)+"f"
            # print 'format_value df=',digits_fraction,'val=',val

            return s % (val)+unit

        else:
            return str(val)+unit


class ArrayConf(ArrayConfMixin, ColConf):
    """
    Column made of numeric array.

    """
    pass


class ListArrayConf(ArrayConfMixin, ColConf):
    """
    Column made of an array of lists.

    """

    def __init__(self, attrname, dtype=None,  **attrs):
        ArrayConfMixin.__init__(self,  attrname, None, dtype=np.object,  **attrs)

    def add(self, ids, values=None):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.zeros(1, self._dtype)
                _values[0] = values

        else:

            _ids = ids
            if values is not None:
                _values = np.zeros(len(ids), self._dtype)
                _values[:] = values

        if values is None:
            _values = self.get_defaults(_ids)

        # print 'add ids, _values',self.attrname,ids
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        newvalue = np.concatenate((self.get_value(), _values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))

        if self._is_index:
            self.add_indices(_ids, _values)

    def add_plugin(self, ids, values=None):
        # print 'add_plugin',self.attrname,ids
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.zeros(1, self._dtype)
                _values[0] = values

        else:

            _ids = ids
            if values is not None:
                _values = np.zeros(len(ids), self._dtype)
                _values[:] = values

        if values is None:
            _values = self.get_defaults(_ids)

        # print 'add ids, _values',self.attrname,ids
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        newvalue = np.concatenate((self.get_value(), _values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))

        if self._is_index:
            self.add_indices(_ids, _values)

        if self.plugin:
            self.plugin.exec_events_ids(EVTADDITEM, _ids)


class NumArrayConf(ArrayConfMixin, ColConf):
    """
    Column made of numeric array.

    """
    # def __init__(self, **attrs):
    #    print 'ColConf',attrs

    def __init__(self, attrname, default,
                 digits_integer=None, digits_fraction=None,
                 minval=None, maxval=None,
                 **attrs):

        self.min = minval
        self.max = maxval
        self.digits_integer = digits_integer
        self.digits_fraction = digits_fraction

        ArrayConfMixin.__init__(self,  attrname, default, metatype='number', **attrs)

    def format_value(self, _id, show_unit=False, show_parentesis=False):
        # TODO: handle also linked ids, etc...
        if show_unit:
            unit = ' '+self.format_unit(show_parentesis)
        else:
            unit = ''
        # return repr(self[_id])+unit

        #self.min = minval
        #self.max = maxval
        #self.digits_integer = digits_integer
        #self.digits_fraction = digits_fraction
        val = self[_id]
        if self.digits_fraction is None:
            digits_fraction = 3
        else:
            digits_fraction = self.digits_fraction

        s = "%."+str(digits_fraction)+"f"
        # print 'format_value df=',digits_fraction,'val=',val

        return s % (val)+unit


class IdsArrayConf(ArrayConfMixin, ColConf):
    """
    Column, where each entry is the id of a single Table. 
    """

    def __init__(self, attrname, tab,  is_index=False, id_default=-1, perm='r', **kwargs):
        self._tab = tab
        ArrayConfMixin.__init__(self,  attrname,
                                id_default,  # default id
                                dtype=np.int32,
                                metatype='id',
                                perm=perm,
                                is_index=is_index,
                                **kwargs
                                )
        self.init_xml()
        # print 'IdsConf.__init__',attrname
        # print '  ',self._tab.xmltag,self._attrconfig_id_tab

    def get_defaults(self, ids):
        # create a list, should work for all types and dimensions
        # default can be scalar or an array of any dimension
        # print '\n\nget_defaults',self.attrname,ids,self.get_default()
        return self.get_default()*np.ones(len(ids), dtype=self._dtype)


# -------------------------------------------------------------------------------
    # copied from IdsConf!!!

    def set_linktab(self, tab):
        self._tab = tab

    def get_linktab(self):
        return self._tab

    def init_xml(self):
        # print 'init_xml',self.attrname,self._tab
        if self._tab.xmltag is not None:
            xmltag_tab, xmltag_item_tab, attrname_id_tab = self._tab.xmltag
            if (attrname_id_tab is None) | (attrname_id_tab is ''):
                self._attrconfig_id_tab = None
            else:
                self._attrconfig_id_tab = getattr(self._tab, attrname_id_tab)  # tab = tabman !

            if not hasattr(self, 'is_xml_include_tab'):
                # this means that entire table rows will be included
                self.is_xml_include_tab = False
            # print '  xmltag_tab, xmltag_item_tab, attrname_id_tab',xmltag_tab, xmltag_item_tab, attrname_id_tab,self.is_xml_include_tab

        else:
            self._attrconfig_id_tab = None
            self.is_xml_include_tab = False

    def write_xml(self, fd, _id, indent=0):
        # print 'write_xml',self.attrname
        if (self.xmltag is not None) & (np.all(self[_id] >= 0)):
            # print 'write_xml',self.attrname, _id,'value',self[_id]
            if self._attrconfig_id_tab is None:
                self._write_xml_value(self[_id], fd)
            elif self.is_xml_include_tab:
                # print '    write table row(s)',self[_id]
                self._tab.write_xml(fd, indent, ids=self[_id],
                                    is_print_begin_end=False)
            else:
                # print '    write id(s)',self[_id]
                self._write_xml_value(self._attrconfig_id_tab[self[_id]], fd)

    def _write_xml_value(self, val, fd):
        # print 'write_xml',self.xmltag,hasattr(val, '__iter__')
        if hasattr(val, '__iter__'):
            if len(val) > 0:
                if hasattr(val[0], '__iter__'):
                    # matrix
                    fd.write(xm.mat(self.xmltag, val))
                else:
                    # list
                    fd.write(xm.arr(self.xmltag, val, self.xmlsep))
            else:
                # empty list
                # fd.write(xm.arr(self.xmltag,val))
                # don't even write empty lists
                pass

        elif type(self._default) in (types.UnicodeType, types.StringType):
            if len(val) > 0:
                fd.write(xm.num(self.xmltag, val))

        else:
            # scalar number or string
            fd.write(xm.num(self.xmltag, val))

    def _getstate_specific(self, state):
        """
        Called by __getstate__ to add/change specific states,
        before returning states.
        To be overridden.
        """
        if self._is_save:
            # if self._is_child:
            #    # OK self.value already set in
            #    pass
            # else:
            #    # remove table reference and create ident
            # print '_getstate_specific',self.attrname
            # print '  self._tab',self._tab
            # print '_getstate_specific',self._tab.ident, self._tab.get_ident_abs()
            state['_tab'] = None
            # try:
            state['_ident_tab'] = self._tab.get_ident_abs()
            # except:
            # print 'WARNING:_getstate_specific',self._tab,self._tab.attrname

    def init_postload_internal(self, man, obj):
        # print 'IdsConf.init_postload_internal',self.attrname,hasattr(self,'value'),self._is_save,self._is_localvalue,'obj:',obj.ident

        AttrConf.init_postload_internal(self, man, obj)
        # print 'IdsConf.init_postload_internal',self.attrname,self.get_value().dtype,self.get_value().dtype == np.int64
        if self.get_value().dtype == np.int64:
            print 'WARNING in init_postload_internal: convert ids array to 32 bit'
            self.set_value(np.array(self.get_value(), dtype=np.int32))
        # if self._is_child:
        #    print '  make sure children get initialized'
        #    print '  call init_postload_internal of',self._tab.ident
        #    self._tab.init_postload_internal(obj)

    def init_postload_external(self):
        # if self._is_child:
        #    # restore normally
        #    AttrConf.init_postload_external(self)
        #    self._tab.init_postload_external()
        # else:

        # Substitute absolute ident with link object.
        # Called from init_postload_external of attrsman during load_obj
        #
        ident_abs = self._ident_tab
        # print 'init_postload_external',self.attrname,ident_abs
        obj = self.get_obj()
        rootobj = obj.get_root()
        # print '  obj,rootobj',obj,rootobj
        linkobj = rootobj.get_obj_from_ident(ident_abs)
        # print '  linkobj',linkobj.ident
        self._tab = linkobj
        self.init_xml()

    def is_modified(self):
        return False

    def set_modified(self, is_modified):
        pass


class IdlistsArrayConf(IdsArrayConf):
    """
    Column, where each entry is a list of ids of a single Table. 
    """

    def __init__(self, attrname, tab, metatype=None, perm='r', **kwargs):
        self._is_index = False
        self._tab = tab
        ArrayConfMixin.__init__(self,  attrname,
                                None,  # default, will be substituted by id list
                                dtype='object',
                                metatype='ids',
                                perm=perm,
                                **kwargs
                                )
        self.init_xml()

    def get_defaults(self, ids):
        # here we initialize with None for reach element
        return np.array(len(ids)*[None, ], self._dtype)

    def add(self, ids, values=None):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.zeros(1, self._dtype)
                _values[0] = values

        else:

            _ids = ids
            _values = np.zeros(len(ids), self._dtype)
            _values[:] = values

        if values is None:
            _values = self.get_defaults(_ids)

        # print 'add ids, _values',self.attrname,ids
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        newvalue = np.concatenate((self.get_value(), _values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))

        if self._is_index:
            self.add_indices(_ids, _values)


class TabIdListArrayConf(ArrayConfMixin, ColConf):
    """
    Column made of an array of lists with (table,id) tupels.
    The tables are linked, and will not be saved.
    """

    def __init__(self, attrname, dtype=None, perm='r',  **attrs):
        ArrayConfMixin.__init__(self, attrname,  None,  # default, will be substituted by (table,id) list
                                dtype='object',
                                metatype='tabidlist',
                                perm=perm, **attrs)

    def add(self, ids, values=None):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.zeros(1, self._dtype)
                _values[0] = values

        else:

            _ids = ids
            _values = np.zeros(len(ids), self._dtype)
            _values[:] = values

        if values is None:
            _values = self.get_defaults(_ids)

        # print 'add ids, _values',self.attrname,ids
        # print '  values',values
        # print '  _values',_values
        # print '  self.get_value()',self.get_value()
        # print '  type(_values),type(self.get_value())',type(_values),type(self.get_value())
        # print '  _values.shape,self.get_value().shape',_values.shape,self.get_value().shape

        newvalue = np.concatenate((self.get_value(), _values))
        # print '  ', type(newvalue),newvalue.dtype
        self.set_value(np.concatenate((self.get_value(), _values)))

        if self._is_index:
            self.add_indices(_ids, _values)

    def add_plugin(self, ids, values=None):
        if not hasattr(ids, '__iter__'):
            _ids = [ids]
            if values is not None:
                _values = np.zeros(1, self._dtype)
                _values[0] = values

        else:

            _ids = ids
            _values = np.zeros(len(ids), self._dtype)
            _values[:] = values

        if values is None:
            _values = self.get_defaults(_ids)

    ###

    def format_value(self, _id, show_unit=False, show_parentesis=False):
        s = ''
        rowlist = self[_id]
        if rowlist is None:
            return s
        # elif (type(rowlist)in STRINGTYPES):
        #    return rowlist
        elif len(rowlist) == 0:
            return s
        elif len(rowlist) == 1:
            tab, ids = rowlist[0]
            return str(tab)+'['+str(ids)+']'
        elif len(rowlist) > 1:
            tab, ids = rowlist[0]
            s = str(tab)+'['+str(ids)+']'
            for tab, ids in rowlist[1:]:
                s += ','+str(tab)+'['+str(ids)+']'
            return s

    def _getstate_specific(self, state):
        """
        Called by __getstate__ to add/change specific states,
        before returning states.
        To be overridden.
        """
        # print '_getstate_specific',self.attrname, self._is_save
        # print '  self.get_value',self.get_value()
        # print len(self.get_value())
        if self._is_save:
            n = len(state['value'])
            state['value'] = None
            _tabidlists_save = n*[None]
            i = 0
            for rowlist in self.get_value():
                if rowlist is not None:
                    rowlist_save = []
                    for tab, ids in rowlist:
                        rowlist_save.append([tab.get_ident_abs(), ids])
                        # print '    tab.get_ident'.get_ident()
                        # print '    appended',[tab.get_ident_abs(), ids]
                    _tabidlists_save[i] = rowlist_save
                # print '  ',i,rowlist_save
                i += 1
            state['_tabidlists_save'] = _tabidlists_save

    def init_postload_external(self):
        # Substitute absolute ident with link object.
        # Called from init_postload_external of attrsman during load_obj
        #
        # print 'init_postload_external',self.attrname, len(self._tabidlists_save)
        #obj = self.get_obj()
        #rootobj = obj.get_root()
        # print '  rootobj',rootobj.ident
        #linkobj = rootobj.get_obj_from_ident(ident_abs)
        # print '  linkobj',linkobj.ident
        #self._tab = linkobj

        # Substitute absolute ident with link object.
        # Called from init_postload_external of attrsman during load_obj
        #
        _tabidlists_save = self._tabidlists_save
        #ident_abs = self._ident_value
        # print 'init_postload_external',self.attrname,_tabids_save
        obj = self.get_obj()
        rootobj = obj.get_root()
        # print '  rootobj',rootobj.ident
        tabidlists = np.zeros(len(_tabidlists_save), dtype=self._dtype)

        i = 0
        for rowlist_save in _tabidlists_save:
            rowlist = []
            # print '  rowlist_save',rowlist_save
            if rowlist_save is not None:
                for tabident, ids in rowlist_save:
                    tab = rootobj.get_obj_from_ident(tabident)
                    # print '  recovered tab',tab.get_ident_abs(), ids
                    rowlist.append([tab, ids])
                tabidlists[i] = rowlist
            else:
                tabidlists[i] = None
            i += 1
        # print '  tabidlists', tabidlists
        self.set_value(tabidlists)

    def is_modified(self):
        return False

    def set_modified(self, is_modified):
        pass


class TabIdsArrayConf(ArrayConfMixin, ColConf):
    """
    Column, where each entry contains a tuple with table object and id. 
    The tables are linked, and will not be saved.
    """

    def __init__(self, attrname,  is_index=False, perm='r', **kwargs):
        self._is_index = is_index
        ArrayConfMixin.__init__(self,  attrname,
                                (None, -1),  # default id
                                dtype=[('ob', object), ('id', np.int)],
                                metatype='tabid',
                                perm=perm,
                                **kwargs
                                )

    def get_defaults(self, ids):
        # create a list, should work for all types and dimensions
        # default can be scalar or an array of any dimension
        # print '\n\nget_defaults',self.attrname,ids,self.get_default()
        return np.zeros(len(ids), dtype=self._dtype)

    def _getstate_specific(self, state):
        """
        Called by __getstate__ to add/change specific states,
        before returning states.
        To be overridden.
        """
        if self._is_save:
            n = len(state['value'])
            state['value'] = None
            _tabids_save = n*[None]
            i = 0
            for tab, ids in self.get_value():
                _tabids_save[i] = [tab.get_ident_abs(), ids]
                i += 1
            state['_tabids_save'] = _tabids_save

    def init_postload_external(self):
        # if self._is_child:
        #    # restore normally
        #    AttrConf.init_postload_external(self)
        #    self._tab.init_postload_external()
        # else:

        # Substitute absolute ident with link object.
        # Called from init_postload_external of attrsman during load_obj
        #
        #ident_abs = self._ident_tab
        # print 'reset_linkobj',self.attrname,ident_abs
        #obj = self.get_obj()
        #rootobj = obj.get_root()
        # print '  rootobj',rootobj.ident
        #linkobj = rootobj.get_obj_from_ident(ident_abs)
        # print '  linkobj',linkobj.ident
        #self._tab = linkobj

        # Substitute absolute ident with link object.
        # Called from init_postload_external of attrsman during load_obj
        #
        _tabids_save = self._tabids_save
        #ident_abs = self._ident_value
        # print 'init_postload_external',self.attrname,_tabids_save
        obj = self.get_obj()
        rootobj = obj.get_root()
        # print '  rootobj',rootobj.ident
        tabids = np.zeros(len(self._tabids_save), dtype=self._dtype)

        i = 0
        for tabident, ids in self._tabids_save:
            tab = rootobj.get_obj_from_ident(tabident)
            # print '  ',tab.get_ident_abs(), ids
            tabids[i] = (tab, ids)
            i += 1

        self.set_value(tabids)

    def is_modified(self):
        return False

    def set_modified(self, is_modified):
        pass


class Arrayman(Tabman):
    """
    Manages all table attributes of an object.

    if argument obj is specified with an instance
    then column attributes are stored under this instance.
    The values of attrname is then directly accessible with

    obj.attrname

    If nothing is specified, then column attribute will be stored under
    the respective config instance of this tab man (self).
    The values of attrname is then directly accessible with

    self.attrname.value

    """

    def __init__(self, obj=None, **kwargs):

        Attrsman.__init__(self, obj, **kwargs)
        self._colconfigs = []
        self._inds = np.zeros((0,), dtype=np.int32)
        self._ids = np.zeros((0,), dtype=np.int32)

    def get_inds(self, ids=None):
        if ids is not None:
            return self._inds[ids]
        else:
            return self._inds[self._ids]

    def get_ids(self, inds=None):
        if inds is not None:
            return self._ids[inds]
        else:
            return self._ids

        return

    def __getitem__(self, args):
        if type(args) == types.TupleType:
            attrnames = args[0]
            ids = args[1]
        else:
            attrnames = args
            ids = self._ids

        if hasattr(attrnames, '__iter__'):
            n_rows = len(ids)
            n_cols = len(attrnames)
            # print '  n_rows',n_rows,'n_cols',n_cols,attrnames
            if (n_rows == 0) | (n_cols == 0):
                return np.array([[]])
            else:
                #attrconf = getattr(self,attrnames[0])
                # TODO: detect dtype by adding numbers or use fields!!!
                dtype = np.float32
                out = np.zeros((n_rows, n_cols), dtype=dtype)
                for i, attrname in zip(xrange(n_cols), attrnames):
                    out[:, i] = getattr(self, attrname)[ids]
                return out
        else:
            return getattr(self, attrnames)[ids]

    def get_ind(self, id):
        return self._inds[id]

    def select_ids(self, mask):
        # print 'select_ids'
        # print '  mask\n=',mask
        # print '  self._ids\n=',self._ids
        # if len(self)>0:
        return np.take(self._ids, np.flatnonzero(mask))
        # else:
        #    return np.zeros(0,int)

    def suggest_id(self, is_zeroid=False):
        """
        Returns a an availlable id.

        Options:
            is_zeroid=True allows id to be zero.

        """
        return self.suggest_ids(1, is_zeroid)[0]

    def format_id(self, id):
        return self.format_ids([id])

    def format_ids(self, ids):
        return ', '.join(np.array(ids, dtype='|S24'))

    def get_id_from_formatted(self, idstr):
        return int(idstr)

    def get_ids_from_formatted(self, idstrs):
        idstrs_arr = idstrs.split(',')
        ids = np.zeros(len(idstrs_arr), dtype=np.float32)
        i = 0
        for idstr in idstrs_arr:
            ids[i] = int[idstr]

        return ids

    def suggest_ids(self, n, is_zeroid=False):
        """
        Returns a list of n availlable ids.
        It returns even a list for n=1. 

        Options:
            is_zeroid=True allows id to be zero.
        """
        # TODO: does always return 1 if is_index is True ?????
        # print 'suggest_ids',n,is_zeroid,self._inds,len(self._inds),self._inds.dtype
        ids_unused_orig = np.flatnonzero(np.less(self._inds, 0))

        if not is_zeroid:
            if len(self._inds) == 0:
                ids_unused = np.zeros(0, dtype=np.int32)
            else:
                # avoid 0 as id:
                # ids_unused=take(ids_unused,flatnonzero(greater(ids_unused,0)))
                # print '  ids_unused_orig',ids_unused_orig,type(ids_unused_orig)
                # print '  len(ids_unused_orig)',len(ids_unused_orig),ids_unused_orig.shape
                # print '  greater(ids_unused_orig,0)',greater(ids_unused_orig,0)
                # print '  len(greater(ids_unused_orig,0))',len(greater(ids_unused_orig,0))
                # print '  flatnonzero(greater(ids_unused_orig,0))',flatnonzero(greater(ids_unused_orig,0))
                # print '  len(flatnonzero(greater(ids_unused_orig,0)))=',len(flatnonzero(greater(ids_unused_orig,0)) )
                ids_unused = ids_unused_orig[np.flatnonzero(np.greater(ids_unused_orig, 0))]
            zid = 1
        else:
            if len(self._inds) == 0:
                ids_unused = np.zeros(0, dtype=np.int32)
            else:
                ids_unused = ids_unused_orig.copy()

            zid = 0

        n_unused = len(ids_unused)
        n_max = len(self._inds)-1
        # print '  ids_unused',ids_unused
        # print '  ids_unused.shape',ids_unused.shape
        # print '  len(ids_unused)',len(ids_unused)
        # print '  n_unused,n_max,zid=',n_unused,n_max,zid

        if n_max < zid:
            # first id generation
            ids = np.arange(zid, n+zid, dtype=np.int32)

        elif n_unused > 0:
            if n_unused >= n:
                ids = ids_unused[:n]
            else:
                # print '  ids_unused',ids_unused
                # print '  from to',n_max+1,n_max+1+n-n_unused
                # print '  arange=',arange(n_max+1,n_max+1+n-n_unused)
                # print '  type(ids_unused)',type(ids_unused)
                # print '  dtype(ids_unused)',ids_unused.dtype
                ids = np.concatenate((ids_unused, np.arange(n_max+1, n_max+1+n-n_unused)))

        else:
            ids = np.arange(n_max+1, n_max+1+n, dtype=np.int32)

        return ids

    def _add_ids(self, ids):
        n = len(ids)
        if n == 0:
            return

        id_max = max(ids)
        id_max_old = len(self._inds)-1
        n_array_old = len(self)

        ids_existing = np.take(ids, np.flatnonzero(np.less(ids, id_max_old)))
        # print '  ids',ids,'id_max_old',id_max_old,'ids_existing',ids_existing

        # check here if ids are still available
        # if np.sometrue(  np.not_equal( np.take(self._inds, ids_existing), -1)  ):
        #    print 'WARNING in create_ids: some ids already in use',ids_existing
        #    return np.zeros(0,int)

        # extend index map with -1 as necessary
        if id_max > id_max_old:
            # print 'ext',-1*ones(id_max-id_max_old)
            self._inds = np.concatenate((self._inds, -1*np.ones(id_max-id_max_old, int)))

        # assign n new indexes to new ids
        ind_new = np.arange(n_array_old, n_array_old+n, dtype=np.int32)

        # print 'ind_new',ind_new
        np.put(self._inds, ids, ind_new)

        # print '  concat ids..',self._ids,ids
        self._ids = np.concatenate((self._ids, ids))

    def add_rows(self, n=None, ids=[], **attrs):

        if n is not None:
            ids = self.suggest_ids(n)
        elif (len(ids) == 0) & (len(attrs) > 0):
            # get number of rows from any value vector provided
            ids = self.suggest_ids(len(attrs.values()[0]))
        elif (n is None) & (len(ids) == 0) & (len(attrs) > 0):
            # nothing given really-> do nothing
            return np.zeros((0), np.int)

        else:
            # ids already given , no ids to create
            pass

        # print 'add_rows ids', ids, type(ids)
        self._add_ids(ids)

        for colconfig in self._colconfigs:
            colconfig.add(ids, values=attrs.get(colconfig.attrname, None))
        if self.plugin:
            self.plugin.exec_events_ids(EVTADDITEM, ids)
        return ids

    def copy_cols(self, attrman2, ids=None):
        # WARNING: this method is unsafe for indexed colums as values are
        # copied and are no longer unique
        # print 'copy_cols'
        if ids is None:
            ids2 = attrman2.get_ids()
        else:
            ids2 = ids
        #ids_new = self.suggest_ids(ids2)
        ids_new = self.add_rows(n=len(ids2))

        for colconfig2 in attrman2._colconfigs:
            if hasattr(self, colconfig2.attrname):
                colconfig = getattr(self, colconfig2.attrname)
                colconfig.set(ids_new, values=colconfig2[ids2].copy())

        return ids_new

    def set_rows(self, ids, **attrs):

        for colconfig in self._colconfigs:
            colconfig.set(ids, values=attrs.get(colconfig.attrname, None))
        if self.plugin:
            self.plugin.exec_events_ids(EVTSETITEM, ids)

    def add_row(self, _id=None, **attrs):
        if _id is None:
            _id = self.suggest_id()

        self._add_ids([_id])
        #ids = self.add_rows(1, **attrs)
        for colconfig in self._colconfigs:
            # print '  add_row',colconfig.attrname,attrs.get(colconfig.attrname, None )
            colconfig.add(_id, values=attrs.get(colconfig.attrname, None))

        if self.plugin:
            self.plugin.exec_events_ids(EVTADDITEM, [id])
        return _id

    def set_row(self, _id, **attrs):
        # if _id  is None:
        # print ' set_row ',self.get_ident(),attrs
        for colconfig in self._colconfigs:  # TODO: run through keys!!!!
            # print '  add_row',_id,colconfig.attrname,attrs.get(colconfig.attrname, None )
            # if attrs.has_key(colconfig.attrname):
            #colconfig.set(_id, values = attrs[colconfig.attrname])
            colconfig.set(_id, values=attrs.get(colconfig.attrname, None))

        if self.plugin:
            self.plugin.exec_events_ids(EVTSETITEM, [id])

    def del_rows(self, ids_del):
        # print '\n\ndel_rows',self.ident,ids_del
        # remaining index
        if self.plugin:
            self.plugin.exec_events_ids(EVTDELITEM, ids_del)

        inds_remain = np.ones(len(self), dtype=np.bool_)
        inds_remain[self._inds[ids_del]] = False
        inds_remain = np.flatnonzero(inds_remain)
        for colconfig in self._colconfigs:
            colconfig._del_rows(ids_del, inds_remain)
        self._ids = self._ids[inds_remain]
        if len(self._ids) == 0:
            n = 0
        else:
            n = np.max(self._ids)
        self._inds = np.zeros(n+1, dtype=np.int32)
        self._inds[self._ids] = np.arange(len(self._ids))

    def del_row(self, _id):
        # print 'del_row',id
        self.del_rows_simple([_id])

    def del_rows_simple(self, ids):
        # print '\n\ndel_rows',self.ident,ids
        # print '  self._ids',self._ids
        # print '  self._inds',self._inds
        # TODO: this could be done in with array methods

        if self.plugin:
            self.plugin.exec_events_ids(EVTDELITEM, ids)
        for _id in ids:
            i = self._inds[_id]
            # print '    id to eliminate _id=',_id
            # print '    index to eliminate i=',i
            for colconfig in self._colconfigs:
                # print '  colconfig',colconfig.attrname,i
                # colconfig.delete_ind(i)
                del colconfig[_id]  # this is universal, also for cm.ColConfigs

            # print '    del from id lookup'
            self._ids = np.concatenate((self._ids[:i], self._ids[i+1:]))
            # print '  ids after cut',self._ids

            # print '    free index',id
            if _id == len(self._inds)-1:
                # id is highest, let's shrink index array by 1
                self._inds = self._inds[:-1]
            else:
                self._inds[_id] = -1

            # get ids of all indexes which are above i
            ids_above = np.flatnonzero(self._inds > i)

            # decrease index from those wich are above the deleted one
            #put(self._inds, ids_above,take(self._inds,ids_above)-1)
            self._inds[ids_above] -= 1

            # print '  inds after cut',self._inds

            # print '    self._inds',self._inds

        # print '  del',ids,' done.'

    def clear_rows(self):
        # attention: overridden by ArrayObjman
        # print 'clear_rows',self.ident

        if self.plugin:
            self.plugin.exec_events_ids(EVTDELITEM, self.get_ids())
        self._ids = []
        self._inds = np.zeros((0,), int)
        self._ids = np.zeros((0,), int)

        for colconfig in self._colconfigs:
            # print 'clear_rows',colconfig.attrname,len(colconfig.get_value())
            colconfig.clear()
            # print '  done',len(colconfig.get_value())


class ArrayObjman(Arrayman, TableMixin):
    """
    Array Object management manages objects with numeric Python arrays
     based columns. Can also handle list and dict based columns. 
    """

    def __init__(self, ident, **kwargs):
        self._init_objman(ident, **kwargs)

    def _init_objman(self, ident, is_plugin=False, **kwargs):
        BaseObjman._init_objman(self, ident, managertype='table', **kwargs)
        Arrayman.__init__(self, is_plugin=is_plugin)
        self.set_attrsman(self)

    def init_postload_internal(self, parent):
        """
        Called after set state.
        Link internal states.
        """
        TableMixin.init_postload_internal(self, parent)
        attrman = self.get_attrsman()

        # this should no longer happen in the future as ind and ids
        # have been formatted propperly
        if attrman._inds.dtype != np.int32:
            print 'WARNING: 64 bit ids and inds...will adjust to 32 bit'
            attrman._inds = np.array(attrman._inds, dtype=np.int32)
            attrman._ids = np.array(attrman._ids, dtype=np.int32)

    def init_postload_external(self):
        """
        Called after set state.
        Link internal states.
        """
        TableMixin.init_postload_external(self)
        self._init_attributes()
        self._init_constants()

    def clear_rows(self):
        if self.plugin:
            self.plugin.exec_events_ids(EVTDELITEM, self.get_ids())
        self._inds = np.zeros((0,), dtype=np.int32)
        self._ids = np.zeros((0,), dtype=np.int32)
        for colconfig in self.get_attrsman()._colconfigs:
            # print 'ArrayObjman.clear_rows',colconfig.attrname,len(colconfig.get_value())
            colconfig.clear()
            # print '  done',len(colconfig.get_value())

    def clear(self):
        # print 'ArrayObjman.clear',self.ident
        # clear/reset scalars
        for attrconfig in self.get_attrsman().get_configs(structs=STRUCTS_SCALAR):
            attrconfig.clear()
        self.clear_rows()
        self._init_constants()
        self.set_modified()

        # print '  check: suggest_ids',self.suggest_ids(5)