~tfardet/nngt-developers

NNGT: Core - More efficient edge attribute access v1 APPLIED

Tanguy Fardet: 1
 Core - More efficient edge attribute access

 11 files changed, 213 insertions(+), 146 deletions(-)
#534261 .build.yml success
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~tfardet/nngt-developers/patches/23508/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH NNGT v1] Core - More efficient edge attribute access Export this patch

Make access to a subset of edge attributes faster by using the underlying library's methods rather than generating the array for all edges every time.
This also reduces the memory footprint.
---
 doc/developer/library_shipping.rst |  2 +-
 nngt/core/graph.py                 | 40 ++++++-------
 nngt/core/gt_graph.py              | 76 ++++++++++++------------
 nngt/core/ig_graph.py              | 62 ++++++++++++-------
 nngt/core/nngt_graph.py            | 36 +++++++++++
 nngt/core/nx_graph.py              | 95 +++++++++++++++---------------
 nngt/io/saving_helpers.py          |  9 ++-
 nngt/lib/connect_tools.py          | 13 +++-
 nngt/lib/graph_helpers.py          |  2 +-
 nngt/lib/rng_tools.py              |  6 +-
 testing/test_attributes.py         | 18 +++---
 11 files changed, 213 insertions(+), 146 deletions(-)

diff --git a/doc/developer/library_shipping.rst b/doc/developer/library_shipping.rst
index b4e7888..e722daf 100755
--- a/doc/developer/library_shipping.rst
+++ b/doc/developer/library_shipping.rst
@@ -45,7 +45,7 @@ to build NNGT
    done

    # Bundle external shared libraries into the wheels
    for whl in wheelhouse/*.whl; do
    for whl in wheelhouse/nngt*.whl; do
        auditwheel repair "$whl" -w wheelhouse/
    done

diff --git a/nngt/core/graph.py b/nngt/core/graph.py
index d5010b4..d99fba8 100644
--- a/nngt/core/graph.py
+++ b/nngt/core/graph.py
@@ -1101,15 +1101,9 @@ class Graph(nngt.core.GraphObject):
        '''
        Attributes of the graph's edges.

        .. versionchanged:: 1.0
            Returns the full dict of edges attributes if called without
            arguments.

        .. versionadded:: 0.8

        Parameters
        ----------
        edge : tuple or list of tuples, optional (default: ``None``)
        edges : tuple or list of tuples, optional (default: ``None``)
            Edge whose attribute should be displayed.
        name : str, optional (default: ``None``)
            Name of the desired attribute.
@@ -1135,18 +1129,17 @@ class Graph(nngt.core.GraphObject):
        :func:`~nngt.Graph.set_node_attribute`
        '''
        if name is not None and edges is not None:
            if isinstance(edges, slice):
                return self._eattr[name][edges]
            elif len(edges):
                return self._eattr[edges][name]
            if len(edges):
                return self._eattr.get_eattr(edges=edges, name=name)

            return np.array([])
        elif name is None and edges is None:
            return {k: self._eattr[k]
                    for k in self._eattr.keys()}
        elif name is None:
            return self._eattr[edges]
        else:
            return self._eattr[name]
            return self._eattr.get_eattr(edges=edges)

        return self._eattr[name]

    def get_node_attributes(self, nodes=None, name=None):
        '''
@@ -1428,14 +1421,15 @@ class Graph(nngt.core.GraphObject):
        if self.is_weighted():
            if edges is None:
                return self._eattr["weight"]
            else:
                if len(edges) == 0:
                    return np.array([])

                return np.asarray(self._eattr[edges]["weight"])
        else:
            size = self.edge_nb() if edges is None else len(edges)
            return np.ones(size)
            if len(edges) == 0:
                return np.array([])

            return self._eattr.get_eattr(edges, "weight")

        size = self.edge_nb() if edges is None else len(edges)

        return np.ones(size)

    def get_delays(self, edges=None):
        '''
@@ -1455,8 +1449,8 @@ class Graph(nngt.core.GraphObject):
        '''
        if edges is None:
            return self._eattr["delay"]
        else:
            return self._eattr[edges]["delay"]

        return self._eattr.get_eattr(edges, "delay")

    def neighbours(self, node, mode="all"):
        '''
diff --git a/nngt/core/gt_graph.py b/nngt/core/gt_graph.py
index 61c74f4..4730c38 100755
--- a/nngt/core/gt_graph.py
+++ b/nngt/core/gt_graph.py
@@ -175,43 +175,6 @@ class _GtEProperty(BaseProperty):

        Edge = g.edge

        if isinstance(name, slice):
            eprop = {}
            for k in self:
                ep = g.edge_properties[k]

                if self._edges_deleted:
                    Edge = g.edge
                    data = [ep[Edge(*e)] for e in self.parent().edges_array]

                    eprop[k] = data[name]
                else:
                    eprop[k] = ep.a[name]

            return eprop
        elif nonstring_container(name):
            # name is and edge or contains edges
            if len(name) == 0:
                return {k: [] for k in self}

            eprop = {}
            if nonstring_container(name[0]):
                # name is a list of edges
                for k in self.keys():
                    tmp = g.edge_properties[k]

                    dtype = super().__getitem__(k)

                    eprop[k] = _to_np_array(
                        [tmp[Edge(*e)] for e in name], dtype)
            else:
                # name is a single edge
                for k in self.keys():
                    eprop[k] = g.edge_properties[k][Edge(*name)]

            return eprop

        # name is a string
        dtype = super().__getitem__(name)

        if self._edges_deleted:
@@ -257,6 +220,45 @@ class _GtEProperty(BaseProperty):

        self._num_values_set[name] = len(value)

    def get_eattr(self, edges, name=None):
        g = self.parent()._graph

        Edge = g.edge

        if nonstring_container(edges[0]):
            # many edges
            if name is None:
                eprop = {}

                for k in self.keys():
                    tmp = g.edge_properties[k]

                    dtype = super().__getitem__(k)

                    eprop[k] = _to_np_array(
                        [tmp[Edge(*e)] for e in edges], dtype)

                return eprop

            tmp = g.edge_properties[name]

            dtype = super().__getitem__(name)

            return _to_np_array([tmp[Edge(*e)] for e in edges], dtype)

        if name is None:
            eprop = {}
            for k in self.keys():
                tmp = g.edge_properties[k]

                eprop[k] = tmp[Edge(*edges)]

            return eprop

        tmp = g.edge_properties[name]

        return tmp[Edge(*edges)]

    def set_attribute(self, name, values, edges=None, last_edges=False):
        '''
        Set the edge property.
diff --git a/nngt/core/ig_graph.py b/nngt/core/ig_graph.py
index 880035d..7a7ba90 100755
--- a/nngt/core/ig_graph.py
+++ b/nngt/core/ig_graph.py
@@ -56,7 +56,9 @@ class _IgNProperty(BaseProperty):

    def __getitem__(self, name):
        g = self.parent()._graph

        dtype = _np_dtype(super(_IgNProperty, self).__getitem__(name))

        return _to_np_array(g.vs[name], dtype=dtype)

    def __setitem__(self, name, value):
@@ -138,25 +140,6 @@ class _IgEProperty(BaseProperty):
    def __getitem__(self, name):
        g = self.parent()._graph

        if isinstance(name, slice):
            eprop = {}
            for k in self.keys():
                dtype = _np_dtype(super(_IgEProperty, self).__getitem__(k))
                eprop[k] = _to_np_array(g.es[k], dtype=dtype)[name]
            return eprop
        elif nonstring_container(name):
            eprop = {}
            if nonstring_container(name[0]):
                eids = [g.get_eid(*e) for e in name]
                for k in self.keys():
                    dtype = _np_dtype(super(_IgEProperty, self).__getitem__(k))
                    eprop[k] = _to_np_array(g.es[k], dtype=dtype)[eids]
            else:
                eid = g.get_eid(*name)
                for k in self.keys():
                    eprop[k] = g.es[k][eid]
            return eprop

        dtype = _np_dtype(super(_IgEProperty, self).__getitem__(name))

        return _to_np_array(g.es[name], dtype=dtype)
@@ -175,6 +158,40 @@ class _IgEProperty(BaseProperty):
            raise InvalidArgument("Attribute does not exist yet, use "
                                  "set_attribute to create it.")

    def get_eattr(self, edges, name=None):
        g = self.parent()._graph

        if nonstring_container(edges[0]):
            # many edges
            eids = [g.get_eid(*e) for e in edges]

            if name is None:
                eprop = {}

                if nonstring_container(name[0]):
                    for k in self.keys():
                        dtype = _np_dtype(super().__getitem__(k))
                        eprop[k] = _to_np_array(
                            [g.es[eid][k] for eid in eids], dtype=dtype)

            dtype = _np_dtype(super().__getitem__(name))
            return _to_np_array([g.es[eid][name] for eid in eids], dtype=dtype)
        elif not nonstring_container(edges):
            raise ValueError("Invalid `edges` entry: {}.".format(edges))

        # single edge
        eid = g.get_eid(*edges)

        if name is None:
            eprop = {}

            for k in self.keys():
                eprop[k] = g.es[eid][k]

            return eprop

        return g.es[eid][name]

    def new_attribute(self, name, value_type, values=None, val=None):
        g = self.parent()._graph

@@ -448,7 +465,10 @@ class _IGraph(GraphInterface):
        '''
        Remove nodes (and associated edges) from the graph.
        '''
        self._graph.delete_vertices(nodes)
        if nodes is None:
            self._graph.delete_vertices()
        else:
            self._graph.delete_vertices(nodes)

        for key in self._nattr:
            self._nattr._num_values_set[key] = self.node_nb()
@@ -596,7 +616,7 @@ class _IGraph(GraphInterface):

    def clear_all_edges(self):
        ''' Remove all edges from the graph '''
        self._graph.delete_edges(None)
        self._graph.delete_edges()
        self._eattr.clear()

    #-------------------------------------------------------------------------#
diff --git a/nngt/core/nngt_graph.py b/nngt/core/nngt_graph.py
index 08d0425..3b5fe37 100644
--- a/nngt/core/nngt_graph.py
+++ b/nngt/core/nngt_graph.py
@@ -197,6 +197,42 @@ class _EProperty(BaseProperty):
                                  "set_attribute to create it.")
        self._num_values_set[name] = len(value)

    def get_eattr(self, edges, name=None):
        g = self.parent()

        eid = g.edge_id

        if nonstring_container(edges[0]):
            # many edges
            if name is None:
                eprop = {}

                for k in self.keys():
                    prop = self.prop[k]

                    dtype = super().__getitem__(k)

                    eprop[k] = _to_np_array(
                        [prop[eid(tuple(e))] for e in edges], dtype)

                return eprop

            prop = self.prop[name]

            dtype = super().__getitem__(name)

            return _to_np_array([prop[eid(tuple(e))] for e in edges], dtype)

        # single edge
        if name is None:
            eprop = {}
            for k in self.keys():
                eprop[k] = self.prop[k][eid(tuple(edges))]

            return eprop

        return self.prop[name][eid(tuple(edges))]

    def set_attribute(self, name, values, edges=None, last_edges=False):
        '''
        Set the edge property.
diff --git a/nngt/core/nx_graph.py b/nngt/core/nx_graph.py
index 3c7669a..b848ef7 100755
--- a/nngt/core/nx_graph.py
+++ b/nngt/core/nx_graph.py
@@ -141,55 +141,18 @@ class _NxEProperty(BaseProperty):
    ''' Class for generic interactions with edge properties (networkx)  '''

    def __getitem__(self, name):
        g     = self.parent()._graph
        edges = None

        if isinstance(name, slice):
            edges = self.parent().edges_array[name]
        elif nonstring_container(name):
            if len(name) == 0:
                return []

            if nonstring_container(name[0]):
                edges = name
            else:
                if len(name) != 2:
                    raise InvalidArgument(
                        "key for edge attribute must be one of the following: "
                        "slice, list of edges, edges or attribute name.")
                return g[name[0]][name[1]]

        if isinstance(name, str):
            dtype = _np_dtype(super(_NxEProperty, self).__getitem__(name))
            eprop = np.empty(g.number_of_edges(), dtype=dtype)

            edges = list(g.edges(data=name))

            if len(edges):
                eids  = np.asarray(list(g.edges(data="eid")))[:, 2]

                for i, eid in enumerate(np.argsort(eids)):
                    eprop[i] = edges[eid][2]

            return eprop

        eprop = {k: [] for k in self.keys()}

        for edge in edges:
            data = g.get_edge_data(edge[0], edge[1])

            if data is None:
                raise ValueError("Edge {} does not exist.".format(edge))
        g = self.parent()._graph

        dtype = _np_dtype(super().__getitem__(name))
        eprop = np.empty(g.number_of_edges(), dtype=dtype)

            for k, v in data.items():
                if k != "eid":
                    eprop[k].append(v)
        edges = list(g.edges(data=name))

        dtype = None
        if len(edges):
            eids = np.asarray(list(g.edges(data="eid")))[:, 2]

        for k, v in eprop.items():
            dtype    = _np_dtype(super(_NxEProperty, self).__getitem__(k))
            eprop[k] = _to_np_array(v, dtype)
            for i, eid in enumerate(np.argsort(eids)):
                eprop[i] = edges[eid][2]

        return eprop

@@ -218,6 +181,46 @@ class _NxEProperty(BaseProperty):
            raise InvalidArgument("Attribute does not exist yet, use "
                                  "set_attribute to create it.")

    def get_eattr(self, edges, name=None):
        g = self.parent()._graph

        if nonstring_container(edges[0]):
            # many edges
            name = self.keys() if name is None else [name]

            eprop = {k: [] for k in self.keys()}

            for edge in edges:
                data = g.get_edge_data(*edge)

                if data is None:
                    raise ValueError(
                        "Edge {} does not exist.".format(edge))

                [eprop[k].append(data[k]) for k in name]

            for k, v in eprop.items():
                dtype    = _np_dtype(super().__getitem__(k))
                eprop[k] = _to_np_array(v, dtype)

            if len(name) == 1:
                return eprop[name[0]]

            return eprop

        # single edge
        data = deepcopy(g.get_edge_data(*edges))

        if not data:
            raise ValueError("Edge {} does not exist.".format(edges))

        if name is None:
            del data["eid"]

            return data

        return data[name]

    def new_attribute(self, name, value_type, values=None, val=None):
        g = self.parent()._graph

diff --git a/nngt/io/saving_helpers.py b/nngt/io/saving_helpers.py
index 903c721..34a7ae8 100755
--- a/nngt/io/saving_helpers.py
+++ b/nngt/io/saving_helpers.py
@@ -35,7 +35,8 @@ def _neighbour_list(graph, separator, secondary, attributes):
    for v1 in range(graph.node_nb()):
        for i, v2 in enumerate(lst_neighbours[v1]):
            str_edge = str(v2)
            eattr    = graph.get_edge_attributes((v1, v2))

            eattr = graph.get_edge_attributes((v1, v2))

            for attr in attributes:
                str_edge += "{}{}".format(secondary, eattr[attr])
@@ -112,8 +113,10 @@ def _gml(graph, *args, **kwargs):
    for i, e in enumerate(edges):
        lst_attr = []

        for k, v in graph.edge_attributes.items():
            lst_attr.append(attr_str.format(key=k, val=v[i]))
        attrs = graph.get_edge_attributes(e)

        for k, v in attrs.items():
            lst_attr.append(attr_str.format(key=k, val=v))

        eattr = "\n".join(lst_attr)

diff --git a/nngt/lib/connect_tools.py b/nngt/lib/connect_tools.py
index cafe5aa..db4d483 100755
--- a/nngt/lib/connect_tools.py
+++ b/nngt/lib/connect_tools.py
@@ -31,8 +31,10 @@ from scipy.spatial.distance import cdist
from numpy.random import randint

import nngt
from nngt.lib import InvalidArgument, nonstring_container
from nngt.lib.logger import _log_message
from .errors import InvalidArgument
from .test_functions import nonstring_container
from .logger import _log_message
from .converters import _np_dtype


logger = logging.getLogger(__name__)
@@ -245,7 +247,12 @@ def _cleanup_edges(g, edges, attributes, duplicates, loops, existing, ignore):
                raise InvalidArgument(
                    "Self-loops are present: {}.".format(edges[~test]))

        new_attr  = {k: np.asarray(v)[test] for v, k in attributes.items()}
        new_attr = {}

        for k, v in attributes.items():
            dtype = _np_dtype(g.get_attribute_type(k))

            new_attr[k] = np.asarray(v, dtype=dtype)[test]
    else:
        # check (also) either duplicates or existing
        new_attr = {key: [] for key in attributes}
diff --git a/nngt/lib/graph_helpers.py b/nngt/lib/graph_helpers.py
index 1ce069b..d730570 100755
--- a/nngt/lib/graph_helpers.py
+++ b/nngt/lib/graph_helpers.py
@@ -34,7 +34,7 @@ from .test_functions import nonstring_container, is_integer

def _edge_prop(prop):
    ''' Return edge property `name` as a distribution dict '''
    if is_integer(prop) or isinstance(prop, np.float):
    if is_integer(prop) or isinstance(prop, float):
        return {"distribution": "constant", "value": prop}
    elif isinstance(prop, dict):
        return prop.copy()
diff --git a/nngt/lib/rng_tools.py b/nngt/lib/rng_tools.py
index de2ebef..8811c6a 100755
--- a/nngt/lib/rng_tools.py
+++ b/nngt/lib/rng_tools.py
@@ -142,7 +142,8 @@ def _make_matrix(graph, ecount, values, elist=None):
    mat_distrib = None
    n = graph.node_nb()
    if elist is not None and graph.edge_nb():
        mat_distrib = ssp.coo_matrix((values,(elist[:,0],elist[:,1])),(n,n))
        mat_distrib = ssp.coo_matrix(
            (values, (elist[:, 0], elist[:, 1])), (n, n))
    else:
        mat_distrib = graph.adjacency_matrix()
        mat_distrib.data = values
@@ -265,8 +266,7 @@ def lin_correlated_distrib(graph, elist=None, correl_attribute="betweenness",
            'Graph has no "distance" edge attribute.'
        if 'distance' not in kwargs:
            if last_edges:
                slc  = slice(-len(elist), None)
                data = graph.get_edge_attributes(slc, 'distance')
                data = graph._eattr['distance'][-len(elist):]
            else:
                data = graph.get_edge_attributes(elist, 'distance')
        else:
diff --git a/testing/test_attributes.py b/testing/test_attributes.py
index d107a1c..52a6eab 100755
--- a/testing/test_attributes.py
+++ b/testing/test_attributes.py
@@ -204,8 +204,9 @@ class TestAttributes(TestBasis):
        # check that all lists are present
        nlists = graph.get_node_attributes(name="nlist")

        self.assertTrue(
            np.all(np.unique(nlists) == np.unique([[], [1], [2], [1, 2]])))
        res = np.unique(np.array([[], [1], [2], [1, 2]], dtype=object))

        self.assertTrue(np.all(np.unique(nlists) == res))

        # check that all nodes from 0 to 48 were updated
        self.assertTrue([] not in nlists[:49].tolist())
@@ -235,8 +236,9 @@ class TestAttributes(TestBasis):
        # check that all lists are present
        elists = graph.get_edge_attributes(name="elist")

        self.assertTrue(
            np.all(np.unique(elists) == np.unique([[], [1], [2], [1, 2]])))
        res = np.unique(np.array([[], [1], [2], [1, 2]], dtype=object))

        self.assertTrue(np.all(np.unique(elists) == res))

        # check that all edges where updated
        eattr1 = graph.get_edge_attributes(name="elist", edges=edges).tolist()
@@ -452,7 +454,7 @@ if not nngt.get_config('mpi'):

    if __name__ == "__main__":
        unittest.main()
        test_str_attr()
        test_delays()
        test_attributes_are_copied()
        test_combined_attr()
        # ~ test_str_attr()
        # ~ test_delays()
        # ~ test_attributes_are_copied()
        # ~ test_combined_attr()
--
2.31.1
builds.sr.ht
NNGT/patches/.build.yml: SUCCESS in 34m49s

[Core - More efficient edge attribute access][0] from [Tanguy Fardet][1]

[0]: https://lists.sr.ht/~tfardet/nngt-developers/patches/23508
[1]: mailto:tanguyfardet@protonmail.com

✓ #534261 SUCCESS NNGT/patches/.build.yml https://builds.sr.ht/~tfardet/job/534261