~tfardet/nngt-developers

NNGT: I/O - Full GraphML support and NN/GML improvements v1 SUPERSEDED

Tanguy Fardet: 1
 I/O - Full GraphML support and NN/GML improvements

 18 files changed, 429 insertions(+), 112 deletions(-)
#535060 .build.yml failed
Export patchset (mbox)
How do I use this?

Copy & paste the following snippet into your terminal to import this patchset into git:

curl -s https://lists.sr.ht/~tfardet/nngt-developers/patches/23533/mbox | git am -3
Learn more about email & git
View this thread in the archives

[PATCH NNGT v1] I/O - Full GraphML support and NN/GML improvements Export this patch

Library independent support for GraphML format (read/write).
Bugfix for GML in corner cases with 0 nodes or edges.
Correction of neighbor list for undirected graphs.
Improved checks for edges attributes.
---
 README.md                        |  12 +-
 nngt/analysis/nngt_functions.py  |  13 +-
 nngt/core/graph.py               |   4 +-
 nngt/core/gt_graph.py            |   4 +
 nngt/core/ig_graph.py            |   7 +-
 nngt/core/nngt_graph.py          |   4 +
 nngt/core/nx_graph.py            |   7 +-
 nngt/generation/connectors.py    |   8 +-
 nngt/io/graph_loading.py         |  17 ++-
 nngt/io/graph_saving.py          |  45 ++++---
 nngt/io/loading_helpers.py       | 201 +++++++++++++++++++++++++------
 nngt/io/saving_helpers.py        | 119 +++++++++++++++++-
 nngt/lib/converters.py           |  12 ++
 setup.py                         |   3 +-
 testing/library_compatibility.py |   1 +
 testing/test_attributes.py       |   8 +-
 testing/test_graphclasses.py     |   2 +-
 testing/test_io.py               |  74 ++++++++----
 18 files changed, 429 insertions(+), 112 deletions(-)

diff --git a/README.md b/README.md
index 8bc0d98..42d091c 100755
--- a/README.md
+++ b/README.md
@@ -49,9 +49,19 @@ NNGT requires Python 3.5+ since version 2.0, and is directly available on Pypi.
To install it, make sure you have a valid Python installation, then do:

```
pip install --user nngt
pip install nngt
```

If you want to use it with advanced geometry, geospatial or other tools, you
can use the various extra to automatically download the relevant dependencies
keep only one of the listed possibilities)

```
pip install nngt[matplotlib|nx|ig|geometry|geospatial]
```

To install all dependencies, use `pip install nngt[full]`.

To use it, once installed, open a Python terminal or script file and type

```python
diff --git a/nngt/analysis/nngt_functions.py b/nngt/analysis/nngt_functions.py
index f6880bd..e99b603 100644
--- a/nngt/analysis/nngt_functions.py
+++ b/nngt/analysis/nngt_functions.py
@@ -30,18 +30,21 @@ import scipy.sparse as ssp
def adj_mat(g, weight=None, mformat="csr"):
    data = None

    num_nodes = g.node_nb()
    num_edges = g.edge_nb()

    if weight in g.edge_attributes:
        data = g.get_edge_attributes(name=weight)
    else:
        data = np.ones(g.edge_nb())
        data = np.ones(num_edges)

    if not g.is_directed():
        data = np.repeat(data, 2)

    edges     = np.array(list(g._graph._edges), dtype=int)
    num_nodes = g.node_nb()
    mat       = ssp.coo_matrix((data, (edges[:, 0], edges[:, 1])),
                               shape=(num_nodes, num_nodes))
    edges = np.array(list(g._graph._edges), dtype=int)
    edges = (edges[:, 0], edges[:, 1]) if num_edges else [[], []]

    mat = ssp.coo_matrix((data, edges), shape=(num_nodes, num_nodes))

    return mat.asformat(mformat)

diff --git a/nngt/core/graph.py b/nngt/core/graph.py
index d99fba8..00aaf63 100644
--- a/nngt/core/graph.py
+++ b/nngt/core/graph.py
@@ -36,7 +36,7 @@ import nngt
import nngt.analysis as na

from nngt import save_to_file
from nngt.io.graph_loading import _load_from_file, _library_load
from nngt.io.graph_loading import _load_from_file, _library_load, di_get_edges
from nngt.io.io_helpers import _get_format
from nngt.io.graph_saving import _as_string
from nngt.lib import InvalidArgument, nonstring_container
@@ -256,7 +256,7 @@ class Graph(nngt.core.GraphObject):
        '''
        fmt = _get_format(fmt, filename)

        if fmt not in ("neighbour", "edge_list", "gml"):
        if fmt not in di_get_edges:
            # only partial support for these formats, relying on backend
            libgraph = _library_load(filename, fmt)

diff --git a/nngt/core/gt_graph.py b/nngt/core/gt_graph.py
index 4730c38..00f8577 100755
--- a/nngt/core/gt_graph.py
+++ b/nngt/core/gt_graph.py
@@ -781,6 +781,10 @@ class _GtGraph(GraphInterface):
            if np.max(edge_list) >= num_nodes:
                raise InvalidArgument("Some nodes do no exist.")

            for k, v in attributes.items():
                assert nonstring_container(v) and len(v) == num_edges, \
                    "One attribute per edge is required."

            # set default values for attributes that were not passed
            _set_default_edge_attributes(self, attributes, num_edges)

diff --git a/nngt/core/ig_graph.py b/nngt/core/ig_graph.py
index 7a7ba90..93599fe 100755
--- a/nngt/core/ig_graph.py
+++ b/nngt/core/ig_graph.py
@@ -507,7 +507,8 @@ class _IGraph(GraphInterface):
        -------
        The new connection or None if nothing was added.
        '''
        attributes = {} if attributes is None else deepcopy(attributes)
        attributes = {} if attributes is None \
                     else {k: [v] for k, v in attributes.items()}

        if source == target:
            if not ignore and not self_loop:
@@ -573,6 +574,10 @@ class _IGraph(GraphInterface):
            if np.max(edge_list) >= self.node_nb():
                raise InvalidArgument("Some nodes do no exist.")

            for k, v in attributes.items():
                assert nonstring_container(v) and len(v) == num_edges, \
                    "One attribute per edge is required."

            # set default values for attributes that were not passed
            _set_default_edge_attributes(self, attributes, num_edges)

diff --git a/nngt/core/nngt_graph.py b/nngt/core/nngt_graph.py
index 3b5fe37..5140f37 100644
--- a/nngt/core/nngt_graph.py
+++ b/nngt/core/nngt_graph.py
@@ -786,6 +786,10 @@ class _NNGTGraph(GraphInterface):
            if np.max(edge_list) >= self.node_nb():
                raise InvalidArgument("Some nodes do no exist.")

            for k, v in attributes.items():
                assert nonstring_container(v) and len(v) == num_edges, \
                    "One attribute per edge is required."

            # check edges
            new_attr = None

diff --git a/nngt/core/nx_graph.py b/nngt/core/nx_graph.py
index e6200a6..43e6d3d 100755
--- a/nngt/core/nx_graph.py
+++ b/nngt/core/nx_graph.py
@@ -688,8 +688,11 @@ class _NxGraph(GraphInterface):
            if np.max(edge_list) >= g.number_of_nodes():
                raise InvalidArgument("Some nodes do no exist.")

            for attr in attributes:
                if "_corr" in attr:
            for k, v in attributes.items():
                assert nonstring_container(v) and len(v) == num_edges, \
                    "One attribute per edge is required."

                if "_corr" in k:
                    raise NotImplementedError("Correlated attributes are not "
                                              "available with networkx.")

diff --git a/nngt/generation/connectors.py b/nngt/generation/connectors.py
index 70234d4..070f058 100644
--- a/nngt/generation/connectors.py
+++ b/nngt/generation/connectors.py
@@ -125,16 +125,20 @@ def connect_nodes(network, sources, targets, graph_model, density=None,

        if isinstance(ww, dict):
            attr['weight'] = _generate_random(len(elist), ww)
        else:
        elif nonstring_container(ww):
            attr['weight'] = ww
        else:
            attr['weight'] = np.full(len(elist), ww)

    if 'delays' in kwargs:
        dd = kwargs['delays']

        if isinstance(ww, dict):
            attr['delay'] = _generate_random(len(elist), dd)
        elif nonstring_container(dd):
            attr['weight'] = dd
        else:
            attr['delay'] = dd
            attr['weight'] = np.full(len(elist), dd)

    if network.is_spatial() and distance:
        attr['distance'] = distance
diff --git a/nngt/io/graph_loading.py b/nngt/io/graph_loading.py
index 3c1cf27..b03d774 100755
--- a/nngt/io/graph_loading.py
+++ b/nngt/io/graph_loading.py
@@ -50,6 +50,8 @@ di_get_edges = {
    "neighbour": _get_edges_neighbour,
    "edge_list": _get_edges_elist,
    "gml": _get_edges_gml,
    "graphml": _get_edges_graphml,
    "xml": _get_edges_graphml,
}


@@ -209,31 +211,34 @@ def _load_from_file(filename, fmt="auto", separator=" ", secondary=";",
    lst_lines, struct, shape, positions = None, None, None, None
    fmt = _get_format(fmt, filename)

    if fmt not in ("neighbour", "edge_list", "gml"):
        return [None]*7
    if fmt not in di_get_edges:
        raise ValueError("Unsupported format: '{}'".format(fmt))

    with open(filename, "r") as filegraph:
        lst_lines = _process_file(filegraph, fmt, separator)

    # notifier lines
    di_notif = _get_notif(lst_lines, notifier, attributes, fmt=fmt,
    di_notif = _get_notif(filename, lst_lines, notifier, attributes, fmt=fmt,
                          atypes=attributes_types)

    # get nodes attributes
    nattr_convertor = _gen_convert(di_notif["node_attributes"],
                                   di_notif["node_attr_types"],
                                   attributes_types=attributes_types)
    di_nattributes = _get_node_attr(di_notif, separator, fmt=fmt,
                                    lines=lst_lines, atypes=attributes_types)
                                    lines=lst_lines, convertor=nattr_convertor)

    # make edges and attributes
    eattributes     = di_notif["edge_attributes"]
    di_eattributes  = {name: [] for name in eattributes}
    di_edge_convert = _gen_convert(di_notif["edge_attributes"],
    eattr_convertor = _gen_convert(di_notif["edge_attributes"],
                                   di_notif["edge_attr_types"],
                                   attributes_types=attributes_types)

    # process file
    edges = di_get_edges[fmt](
        lst_lines, eattributes, ignore, notifier, separator, secondary,
        di_attributes=di_eattributes, di_convert=di_edge_convert,
        di_attributes=di_eattributes, convertor=eattr_convertor,
        di_notif=di_notif)

    if cleanup:
diff --git a/nngt/io/graph_saving.py b/nngt/io/graph_saving.py
index 5d062be..c538936 100755
--- a/nngt/io/graph_saving.py
+++ b/nngt/io/graph_saving.py
@@ -40,8 +40,9 @@ from nngt.lib.logger import _log_message

from ..geometry import Shape, _shapely_support
from .io_helpers import _get_format
from .saving_helpers import (_neighbour_list, _edge_list, _gml, _custom_info,
                           _gml_info, _str_bytes_len)
from .saving_helpers import (_neighbour_list, _edge_list, _gml, _xml,
                             _custom_info, _gml_info, _xml_info,
                             _str_bytes_len)


logger = logging.getLogger(__name__)
@@ -54,11 +55,15 @@ logger = logging.getLogger(__name__)
di_format = {
    "neighbour": _neighbour_list,
    "edge_list": _edge_list,
    "gml": _gml
    "gml": _gml,
    "graphml": _xml,
    "xml": _xml,
}

format_graph_info = defaultdict(lambda: _custom_info)
format_graph_info["gml"] = _gml_info
format_graph_info["xml"] = _xml_info
format_graph_info["graphml"] = _xml_info


# --------------- #
@@ -233,26 +238,27 @@ def _as_string(graph, fmt="neighbour", separator=" ", secondary=";",
    }

    # add node attributes to the notifications
    for nattr in additional_notif["node_attributes"]:
        key = "na_" + nattr
    if fmt != "graphml":
        for nattr in additional_notif["node_attributes"]:
            key = "na_" + nattr

        tmp = np.array2string(
            graph.get_node_attributes(name=nattr), max_line_width=np.NaN,
            separator=separator)[1:-1].replace("'" + separator + "'",
                                               '"' + separator + '"')
            tmp = np.array2string(
                graph.get_node_attributes(name=nattr), max_line_width=np.NaN,
                separator=separator)[1:-1].replace("'" + separator + "'",
                                                   '"' + separator + '"')

        # replace possible variants
        tmp = tmp.replace("'" + separator + '"', '"' + separator + '"')
        tmp = tmp.replace('"' + separator + "'", '"' + separator + '"')
            # replace possible variants
            tmp = tmp.replace("'" + separator + '"', '"' + separator + '"')
            tmp = tmp.replace('"' + separator + "'", '"' + separator + '"')

        if tmp.startswith("'"):
            tmp = '"' + tmp[1:]
            if tmp.startswith("'"):
                tmp = '"' + tmp[1:]

        if tmp.endswith("'"):
             tmp = tmp[:-1] + '"'
            if tmp.endswith("'"):
                 tmp = tmp[:-1] + '"'

        # make and store final string
        additional_notif[key] = tmp
            # make and store final string
            additional_notif[key] = tmp

    # save positions for SpatialGraph (and shape if Shapely is available)
    if graph.is_spatial():
@@ -308,7 +314,8 @@ def _as_string(graph, fmt="neighbour", separator=" ", secondary=";",
            g._net    = weakref.ref(graph)

    str_graph = di_format[fmt](graph, separator=separator,
                               secondary=secondary, attributes=attributes)
                               secondary=secondary, attributes=attributes,
                               additional_notif=additional_notif)

    # set numpy cut threshold back on
    np.set_printoptions(threshold=old_threshold)
diff --git a/nngt/io/loading_helpers.py b/nngt/io/loading_helpers.py
index f1ad4d4..c07b18d 100755
--- a/nngt/io/loading_helpers.py
+++ b/nngt/io/loading_helpers.py
@@ -23,13 +23,14 @@

""" Loading helpers """

from collections import defaultdict
import re
import types

import numpy as np

from ..lib.converters import (_np_dtype, _to_int, _to_string, _to_list,
                              _string_from_object)
                              _string_from_object, _python_type)


__all__ = [
@@ -37,6 +38,7 @@ __all__ = [
    "_gen_convert",
    "_get_edges_elist",
    "_get_edges_gml",
    "_get_edges_graphml",
    "_get_edges_neighbour",
    "_get_node_attr",
    "_get_notif",
@@ -62,13 +64,13 @@ def _process_file(f, fmt, separator):
            elif clean_line.endswith("]") and len(clean_line) > 1:
                lines.append(clean_line[:-1].strip())
                lines.append("]")
            else:
            elif clean_line:
                lines.append(clean_line)

        return lines

    # otherwise just cleanup the lines
    return [_cleanup_line(line, separator) for line in f.readlines()]
    return [_cleanup_line(line, separator) for line in f.readlines() if line]


# ---------------- #
@@ -94,13 +96,13 @@ def _format_notif(notif_name, notif_val):
        return notif_val


def _get_notif(lines, notifier, attributes, fmt=None, atypes=None):
def _get_notif(filename, lines, notifier, attributes, fmt=None, atypes=None):
    di_notif = {
        "node_attributes": [], "edge_attributes": [], "node_attr_types": [],
        "edge_attr_types": [],
    }

    # special case for GML
    # special cases for GML and GraphML
    if fmt == "gml":
        start = 0

@@ -109,20 +111,32 @@ def _get_notif(lines, notifier, attributes, fmt=None, atypes=None):
                start = i
                break

        # nodes
        nodes = [i for i, l in enumerate(lines) if l == "node" and i > start]
        edges = [i for i, l in enumerate(lines) if l == "edge" and i > start]

        num_nodes = len(nodes)
        num_edges = len(edges)

        # nodes
        di_notif["size"]  = num_nodes
        di_notif["nodes"] = nodes

        # node attributes
        diff = np.diff(nodes) - 4  # number of lines other than node spec

        num_nattr = diff[0]
        num_nattr = 0

        if not np.all(diff == num_nattr):
            raise RuntimeError("All nodes should have the same attributes.")
        if num_nodes > 1:
            num_nattr = diff[0]

            if not np.all(diff == num_nattr):
                raise RuntimeError(
                    "All nodes should have the same attributes.")
        elif num_nodes:
            if num_edges:
                num_nattr = edges[0] - nodes[0] - 4
            else:
                num_nattr = len(lines) - nodes[0] - 5

        if num_nattr > len(di_notif["node_attributes"]):
            for i in range(nodes[0] + 3, nodes[0] + num_nattr + 3):
@@ -143,16 +157,20 @@ def _get_notif(lines, notifier, attributes, fmt=None, atypes=None):
            di_notif[key] = _format_notif(key, val)

        # edges
        edges = [i for i, l in enumerate(lines) if l == "edge" and i > start]

        di_notif["edges"] = edges

        diff = np.diff(edges) - 5  # number of lines other than edge spec

        num_eattr = diff[0]
        num_eattr = 0

        if not np.all(diff == num_eattr):
            raise RuntimeError("All edges should have the same attributes.")
        if num_edges > 1:
            num_eattr = diff[0]

            if not np.all(diff == num_eattr):
                raise RuntimeError(
                    "All edges should have the same attributes.")
        elif num_edges == 1:
            num_eattr = len(lines) - edges[0] - 6

        if num_eattr > len(di_notif["edge_attributes"]):
            for i in range(edges[0] + 4, edges[0] + num_eattr + 4):
@@ -164,7 +182,65 @@ def _get_notif(lines, notifier, attributes, fmt=None, atypes=None):
                        _string_from_object(atypes.get(name, object)))
                else:
                    di_notif["edge_attr_types"].append("object")
    elif fmt == "graphml" or fmt == "xml":
        try:
            from lxml import etree as ET
            lxml = True
        except:
            lxml = False
            import xml.etree.ElementTree as ET
            from io import StringIO

        root = ET.parse(filename).getroot()

        ns = root.nsmap if lxml else dict([
                node for _, node in ET.iterparse(filename, events=['start-ns'])
             ])

        di_notif["namespace"] = ns

        graph = root.find("graph", ns)

        di_notif["nodes"] = list(graph.findall("node", ns))
        di_notif["edges"] = list(graph.findall("edge", ns))

        # graph properties
        for elt in root.findall("data", ns):
            di_notif[elt.get("key")] = elt.text

        for elt in graph.findall("data", ns):
            di_notif[elt.get("key")] = elt.text

        if "size" in di_notif:
            di_notif["size"] = int(di_notif["size"])
        else:
            di_notif["size"] = len(di_notif["nodes"])

        # directedness
        di_notif["directed"] = \
            True if graph.get("edgedefault") == "directed" else False

        # node and edge attributes
        di_notif["node_attributes"] = []
        di_notif["edge_attributes"] = []
        di_notif["node_attr_types"] = []
        di_notif["edge_attr_types"] = []
        di_notif["nattr_keytoname"] = {}
        di_notif["eattr_keytoname"] = {}

        for elt in root.findall("./key", ns):
            if elt.get("for") == "node":
                di_notif["node_attributes"].append(elt.get("attr.name"))
                di_notif["node_attr_types"].append(elt.get("attr.type"))
                di_notif["nattr_keytoname"][elt.get("id")] = \
                    elt.get("attr.name")
            elif elt.get("for") == "edge":
                di_notif["edge_attributes"].append(elt.get("attr.name"))
                di_notif["edge_attr_types"].append(elt.get("attr.type"))
                di_notif["eattr_keytoname"][elt.get("id")] = \
                    elt.get("attr.name")
    else:
        # edge list and neighbour formatting
        for line in lines:
            if line.startswith(notifier):
                idx_eq = line.find("=")
@@ -192,7 +268,7 @@ def _get_notif(lines, notifier, attributes, fmt=None, atypes=None):
# ----- #

def _get_edges_neighbour(lst_lines, attributes, ignore, notifier, separator,
                         secondary, di_attributes, di_convert, **kwargs):
                         secondary, di_attributes, convertor, **kwargs):
    '''
    Add edges and attributes to `edges` and `di_attributes` for the "neighbour"
    format.
@@ -223,15 +299,15 @@ def _get_edges_neighbour(lst_lines, attributes, ignore, notifier, separator,
                        attr_val = content[1:] if len(content) > 1 else []

                        for name, val in zip(attributes, attr_val):
                            di_attributes[name].append(di_convert[name](val))
                            di_attributes[name].append(convertor[name](val))

    return edges


def _get_edges_elist(lst_lines, attributes, ignore, notifier, separator,
                     secondary, di_attributes, di_convert, **kwargs):
                     secondary, di_attributes, convertor, **kwargs):
    '''
    Add edges and attributes to `edges` and `di_attributes` for the "neighbour"
    Add edges and attributes to `edges` and `di_attributes` for the edge list
    format.
    '''
    edges = []
@@ -250,41 +326,77 @@ def _get_edges_elist(lst_lines, attributes, ignore, notifier, separator,
            if len(data) == 3 and secondary in data[2]:  # secondary notifier
                attr_data = data[2].split(secondary)
                for name, val in zip(attributes, attr_data):
                    di_attributes[name].append(di_convert[name](val))
                    di_attributes[name].append(convertor[name](val))
            elif len(data) == len(attributes) + 2:  # regular columns
                for name, val in zip(attributes, data[2:]):
                    di_attributes[name].append(di_convert[name](val))
                    di_attributes[name].append(convertor[name](val))

    return edges


def _get_edges_gml(lst_lines, attributes, *args, di_attributes=None,
                   di_convert=None, di_notif=None):
                   convertor=None, di_notif=None):
    '''
    Add edges and attributes to `edges` and `di_attributes` for the "neighbour"
    format.
    Add edges and attributes to `edges` and `di_attributes` for the gml format.
    '''
    edges = []

    edge_lines = di_notif["edges"]
    num_eattr  = len(di_attributes)

    for line_num in edge_lines:
    for line_num in di_notif["edges"]:
        source = int(lst_lines[line_num + 2][7:])
        target = int(lst_lines[line_num + 3][7:])

        edges.append((source, target))

        for i, name in zip(range(num_eattr), attributes):
        for i, name in enumerate(attributes):
            lnum  = line_num + 4 + i
            start = lst_lines[lnum].find(" ") + 1
            attr  = lst_lines[lnum][start:]
            di_attributes[name].append(di_convert[name](attr))
            di_attributes[name].append(convertor[name](attr))

    return edges


def _get_edges_graphml(lst_lines, attributes, *args, di_attributes=None,
                       convertor=None, di_notif=None):
    '''
    Add edges and attributes to `edges` and `di_attributes` for the graphml
    format.
    '''
    edges = []

    num_eattr = len(di_attributes)

    ns = di_notif["namespace"]

    try:
        int(di_notif["edges"][0].source)
        ids = True
    except:
        ids = False
        nid = {elt.get("id"): i for i, elt in enumerate(di_notif["nodes"])}

    for elt in di_notif["edges"]:
        if ids:
            source = int(elt.get("source"))
            target = int(elt.get("target"))
        else:
            source = int(nid[elt.get("source")])
            target = int(nid[elt.get("target")])

        edges.append((source, target))

        key_to_name = di_notif["eattr_keytoname"]

        for attr in elt.findall("data", ns):
            name = key_to_name[attr.get("key")]
            di_attributes[name].append(convertor[name](attr.text))

    return edges


def _get_node_attr(di_notif, separator, fmt=None, lines=None, atypes=None):
def _get_node_attr(di_notif, separator, fmt=None, lines=None, convertor=None):
    '''
    Return node attributes.

@@ -293,14 +405,12 @@ def _get_node_attr(di_notif, separator, fmt=None, lines=None, atypes=None):

    For GML, need to get them from the nodes.
    '''
    di_nattr   = {}
    di_nattr = {}

    if fmt == "gml":
        node_lines = di_notif["nodes"]
        num_nattr  = node_lines[1] - node_lines[0] - 3  # lines other than attr

        has_types = len(di_notif["node_attr_types"]) == num_nattr

        if num_nattr:
            for line_num in node_lines:
                for i in range(num_nattr):
@@ -314,13 +424,20 @@ def _get_node_attr(di_notif, separator, fmt=None, lines=None, atypes=None):
                    if name not in di_nattr:
                        di_nattr[name] = []

                    dtype = str if atypes is None else atypes.get(name, str)
                    di_nattr[name].append(convertor[name](val))
    elif fmt == "graphml" or fmt == "xml":
        ns = di_notif["namespace"]
        key_to_name = di_notif["nattr_keytoname"]

                    if has_types:
                        dtype = _type_converter(di_notif["node_attr_types"][i])
        for elt in di_notif["nodes"]:
            for attr in elt.findall("data", ns):
                name = key_to_name[attr.get("key")]
                if name not in di_nattr:
                    di_nattr[name] = []

                    di_nattr[name].append(dtype(val))
                di_nattr[name].append(convertor[name](attr.text))
    else:
        # edge list and neighbors formatting
        nattr_name = {str("na_" + k): k for k in di_notif["node_attributes"]}
        nattr_type = di_notif["node_attr_types"]

@@ -358,7 +475,7 @@ def _gen_convert(attributes, attr_types, attributes_types=None):
    Generate a conversion dictionary that associates the right type to each
    attribute
    '''
    di_convert = {}
    di_convert = defaultdict(lambda: (lambda x: x))

    if attributes and not attr_types:
        attr_types.extend(("string" for _ in attributes))
@@ -370,14 +487,18 @@ def _gen_convert(attributes, attr_types, attributes_types=None):
        elif attr_type in ("double", "float", "real"):
            di_convert[attr] = float
        elif attr_type in ("str", "string"):
            di_convert[attr] = lambda x: str(x).strip("\"'")
            def string_convertor(s):
                if s:
                    start, end = s[0], s[-1]
                    if start == end and start in ("'", '"'):
                        return s[1:-1]
                return s
            di_convert[attr] = lambda x: string_convertor(x)
        elif attr_type in ("int", "integer"):
            di_convert[attr] = _to_int
        elif attr_type in ("lst", "list", "tuple", "array"):
            di_convert[attr] = _to_list
        elif attr_type == "object":
            di_convert[attr] = lambda x: x
        else:
        elif attr_type != "object":
            raise TypeError("Invalid attribute type: '{}'.".format(attr_type))

    return di_convert
diff --git a/nngt/io/saving_helpers.py b/nngt/io/saving_helpers.py
index 34a7ae8..bd6e550 100755
--- a/nngt/io/saving_helpers.py
+++ b/nngt/io/saving_helpers.py
@@ -23,14 +23,27 @@

""" IO tools for NNGT """

import logging

def _neighbour_list(graph, separator, secondary, attributes):
from nngt.lib.logger import _log_message

logger = logging.getLogger(__name__)


def _neighbour_list(graph, separator, secondary, attributes, **kwargs):
    '''
    Generate a string containing the neighbour list of the graph as well as a
    dict containing the notifiers as key and the associated values.
    @todo: speed this up!
    '''
    lst_neighbours = list(graph.adjacency_matrix(mformat="lil").rows)
    lst_neighbours = None

    if graph.is_directed():
        lst_neighbours = list(graph.adjacency_matrix(mformat="lil").rows)
    else:
        import scipy.sparse as ssp
        lst_neighbours = list(
            ssp.tril(graph.adjacency_matrix(), format="lil").rows)

    for v1 in range(graph.node_nb()):
        for i, v2 in enumerate(lst_neighbours[v1]):
@@ -51,7 +64,7 @@ def _neighbour_list(graph, separator, secondary, attributes):
    return str_neighbours


def _edge_list(graph, separator, secondary, attributes):
def _edge_list(graph, separator, secondary, attributes, **kwargs):
    ''' Generate a string containing the edge list and their properties. '''
    edges = graph.edges_array

@@ -129,8 +142,99 @@ def _gml(graph, *args, **kwargs):
    return str_gml


def _xml(graph, attributes, **kwargs):
    pass
def _xml(graph, attributes=None, additional_notif=None, **kwargs):
    try:
        from lxml import etree as ET
        lxml = True
    except:
        lxml = False
        import xml.etree.ElementTree as ET
        _log_message(logger, "WARNING",
                     "LXML is not installed, using Python XML for export. "
                     "Some apps like Gephi <= 0.9.2 will not read attributes "
                     "from the generated GraphML file due to elements' order.")

    NS_GRAPHML = "http://graphml.graphdrawing.org/xmlns"
    NS_XSI = "http://www.w3.org/2001/XMLSchema-instance"
    NS_Y = "http://www.yworks.com/xml/graphml"
    NSMAP = {
        "xsi": NS_XSI
    }
    SCHEMALOCATION = " ".join(
        [
            "http://graphml.graphdrawing.org/xmlns",
            "http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd",
        ]
    )

    doc = ET.Element(
        "graphml",
        {
            "xmlns": NS_GRAPHML,
        },
        nsmap=NSMAP
    )

    n = doc.set("{{}}xsi".format(NS_GRAPHML), NS_XSI)
    n = doc.set("{{}}schemaLocation".format(NS_XSI), SCHEMALOCATION)

    # make graph element
    directedness = "directed" if graph.is_directed() else "undirected"

    eg = ET.SubElement(doc, "graph", edgedefault=directedness, id=graph.name)

    # prepare graph data
    del additional_notif["directed"]
    del additional_notif["name"]

    nattrs = additional_notif.pop("node_attributes")
    ntypes = additional_notif.pop("node_attr_types")

    for attr, atype in zip(nattrs, ntypes):
        kw = {"for": "node", "attr.name": attr, "attr.type": atype}
        if lxml:
            key = ET.Element("key", id=attr, **kw)
            eg.addprevious(key)
        else:
            ET.SubElement(doc, "key", id=attr, **kw)

    eattrs = additional_notif.pop("edge_attributes")
    etypes = additional_notif.pop("edge_attr_types")

    for attr, atype in zip(eattrs, etypes):
        kw = {"for": "edge", "attr.name": attr, "attr.type": atype}
        if lxml:
            key = ET.Element("key", id=attr, **kw)
            eg.addprevious(key)
        else:
            ET.SubElement(doc, "key", id=attr, **kw)

    # add remaining information as data to the graph
    for k, v in additional_notif.items():
        elt = ET.SubElement(doc, "data", key=k)
        elt.text = str(v)

    # add node information
    nattr = graph.get_node_attributes()

    for n in graph.get_nodes():
        nelt = ET.SubElement(eg, "node", id=str(n))

        for k, v in nattr.items():
            elt = ET.SubElement(nelt, "data", key=k)
            elt.text = str(v[n])

    # add edge information
    for e in graph.get_edges():
        nelt = ET.SubElement(eg, "edge", id="e{}".format(graph.edge_id(e)),
                             source=str(e[0]), target=str(e[1]))
        for k in eattrs:
            elt = ET.SubElement(nelt, "data", key=k)
            elt.text = str(graph.get_edge_attributes(e, name=k))

    kw = {"pretty_print": True} if lxml else {}

    return ET.tostring(doc, encoding="unicode", **kw)


def _gt(graph, attributes, **kwargs):
@@ -160,5 +264,10 @@ def _gml_info(graph_info, *args, **kwargs):
    return info_str


def _xml_info(*args, **kwargs):
    ''' Return empty string '''
    return ""


def _str_bytes_len(s):
    return len(s.encode('utf-8'))
diff --git a/nngt/lib/converters.py b/nngt/lib/converters.py
index 9a02126..041e371 100755
--- a/nngt/lib/converters.py
+++ b/nngt/lib/converters.py
@@ -105,6 +105,18 @@ def _np_dtype(attribute_type):
    return object


def _python_type(attribute_type):
    '''
    Return a relevant numpy dtype entry.
    '''
    if attribute_type in ("double", "float", "real"):
        return float
    elif attribute_type in ("int", "integer"):
        return int

    return str


def _type_converter(attribute_type):
    if not isinstance(attribute_type, str):
        return attribute_type
diff --git a/setup.py b/setup.py
index 8a6fd14..b4dcf50 100755
--- a/setup.py
+++ b/setup.py
@@ -168,7 +168,8 @@ setup_params = dict(
        'geometry': ['matplotlib', 'shapely', 'dxfgrabber', 'svg.path'],
        'geospatial': ['matplotlib', 'geopandas', 'descartes', 'cartopy'],
        'full': ['cython', 'networkx>=2.4', 'shapely', 'dxfgrabber',
                 'svg.path', 'matplotlib', 'geopandas', 'descartes', 'cartopy']
                 'svg.path', 'matplotlib', 'geopandas', 'descartes', 'cartopy',
                 'lxml']
    },

    # Cython module
diff --git a/testing/library_compatibility.py b/testing/library_compatibility.py
index 757563f..cac98f3 100644
--- a/testing/library_compatibility.py
+++ b/testing/library_compatibility.py
@@ -140,6 +140,7 @@ def test_assortativity():

    # UNDIRECTED
    edge_list = [(0, 3), (1, 0), (1, 2), (2, 4), (4, 1), (4, 3)]
    weights = weights[:len(edge_list)]

    # expected results
    assort_unweighted = -0.33333333333333215
diff --git a/testing/test_attributes.py b/testing/test_attributes.py
index 52a6eab..a958001 100755
--- a/testing/test_attributes.py
+++ b/testing/test_attributes.py
@@ -454,7 +454,7 @@ if not nngt.get_config('mpi'):

    if __name__ == "__main__":
        unittest.main()
        # ~ test_str_attr()
        # ~ test_delays()
        # ~ test_attributes_are_copied()
        # ~ test_combined_attr()
        test_str_attr()
        test_delays()
        test_attributes_are_copied()
        test_combined_attr()
diff --git a/testing/test_graphclasses.py b/testing/test_graphclasses.py
index 6e2cfd0..af7aa09 100755
--- a/testing/test_graphclasses.py
+++ b/testing/test_graphclasses.py
@@ -119,7 +119,7 @@ def test_structure_graph():
        d2 = 5
        ng.connect_groups(g, room2, room3, "erdos_renyi", avg_deg=d2)
        ng.connect_groups(g, room2, room4, "erdos_renyi", avg_deg=d2,
                                       weights=2)
                          weights=2)

        d3 = 20
        ng.connect_groups(g, room3, room1, "erdos_renyi", avg_deg=d3)
diff --git a/testing/test_io.py b/testing/test_io.py
index 2a3ea9d..c5bc047 100644
--- a/testing/test_io.py
+++ b/testing/test_io.py
@@ -29,6 +29,10 @@ from tools_testing import foreach_graph
current_dir = os.path.dirname(os.path.abspath(__file__)) + '/'
error = 'Wrong {{val}} for {graph}.'

formats = ("neighbour", "edge_list", "gml", "graphml")

filetypes = ("nn", "el", "gml", "graphml")

gfilename = current_dir + 'g.graph'


@@ -64,8 +68,8 @@ class TestIO(TestBasis):
            except:
                pass
        try:
            for fmt in ("nn", "el", "gml"):
                os.remove(current_dir + 'test.' + fmt)
            for ft in filetypes:
                os.remove(current_dir + 'test.' + ft)
        except:
            pass

@@ -77,11 +81,6 @@ class TestIO(TestBasis):
    def gen_graph(self, graph_name):
        # check whether we are loading from file
        if "." in graph_name:
            with_nngt = nngt.get_config("backend") == "nngt"

            if "graphml" in graph_name and with_nngt:
                return None

            abspath = network_dir + graph_name
            di_instructions = self.parser.get_graph_options(graph_name)
            graph = nngt.Graph.from_file(abspath, **di_instructions,
@@ -170,9 +169,9 @@ class TestIO(TestBasis):

        old_edges = g.edges_array

        for fmt in ("nn", "el", "gml"):
            g.to_file(current_dir + 'test.' + fmt)
            h = nngt.Graph.from_file(current_dir + 'test.' + fmt)
        for ft in filetypes:
            g.to_file(current_dir + 'test.' + ft)
            h = nngt.Graph.from_file(current_dir + 'test.' + ft)

            # for neighbour list, we need to give the edge list to have
            # the edge attributes in the same order as the original graph
@@ -181,10 +180,10 @@ class TestIO(TestBasis):
                                                         name="test_attr"))
            if not allclose:
                print("Results differed for '{}'.".format(g.name))
                print("using file 'test.{}'.".format(fmt))
                print("using file 'test.{}'.".format(ft))
                print(g.get_edge_attributes(name="test_attr"))
                print(h.get_edge_attributes(edges=old_edges, name="test_attr"))
                with open(current_dir + 'test.' + fmt, 'r') as f:
                with open(current_dir + 'test.' + ft, 'r') as f:
                    for line in f.readlines():
                        print(line.strip())

@@ -197,7 +196,7 @@ def test_empty_out_degree():

    g.new_edge(0, 1)

    for fmt in ("neighbour", "edge_list"):
    for fmt in formats:
        nngt.save_to_file(g, gfilename, fmt=fmt)

        h = nngt.load_from_file(gfilename, fmt=fmt)
@@ -217,7 +216,7 @@ def test_str_attributes():
    g.new_node_attribute("rnd", "string")
    g.set_node_attribute("rnd", values=["s'adf", 'sd fr"'])

    for fmt in ("neighbour", "edge_list"):
    for fmt in formats:
        nngt.save_to_file(g, gfilename, fmt=fmt)

        h = nngt.load_from_file(gfilename, fmt=fmt)
@@ -245,20 +244,46 @@ def test_structure():

    g = nngt.Graph(structure=struct)

    g.to_file(gfilename, fmt="edge_list")
    for fmt in formats:
        g.to_file(gfilename, fmt=fmt)

    h = nngt.load_from_file(gfilename, fmt="edge_list")
        h = nngt.load_from_file(gfilename, fmt=fmt)

    assert g.structure == h.structure
        assert g.structure == h.structure

    # with a neuronal population
    g = nngt.Network.exc_and_inhib(100)

    g.to_file(gfilename, fmt="edge_list")
    for fmt in formats:
        g.to_file(gfilename, fmt=fmt)

        h = nngt.load_from_file(gfilename, fmt=fmt)

        assert g.population == h.population


    h = nngt.load_from_file(gfilename, fmt="edge_list")
@pytest.mark.mpi_skip
def test_spatial():
    from nngt.geometry import Shape

    shape = Shape.disk(100, default_properties={"plop": 0.2, "height": 1.})
    area = Shape.rectangle(10, 10, default_properties={"height": 10.})
    shape.add_area(area, name="center")

    g = nngt.SpatialGraph(20, shape=shape)

    assert g.population == h.population
    for fmt in formats:
        g.to_file(gfilename, fmt=fmt)

        h = nngt.load_from_file(gfilename, fmt=fmt)

        assert np.all(np.isclose(g.get_positions(), h.get_positions()))
        assert g.shape.almost_equals(h.shape)

        for name, area in g.shape.areas.items():
            assert area.almost_equals(h.shape.areas[name])

            assert area.properties == h.shape.areas[name].properties


@pytest.mark.mpi_skip
@@ -268,11 +293,13 @@ def test_node_attributes():

    g.new_node_attribute("size", "int", [2*(i+1) for i in range(num_nodes)])

    g.to_file(gfilename, fmt="edge_list")
    for fmt in formats:
        g.to_file(gfilename, fmt=fmt)

    h = nngt.load_from_file(gfilename, fmt="edge_list")
        h = nngt.load_from_file(gfilename, fmt=fmt)

    assert np.array_equal(g.node_attributes["size"], h.node_attributes["size"])
        assert np.array_equal(g.node_attributes["size"],
                              h.node_attributes["size"])


# ---------- #
@@ -287,4 +314,5 @@ if __name__ == "__main__":
        test_str_attributes()
        test_structure()
        test_node_attributes()
        test_spatial()
        unittest.main()
--
2.31.1
builds.sr.ht
NNGT/patches/.build.yml: FAILED in 22s

[I/O - Full GraphML support and NN/GML improvements][0] from [Tanguy Fardet][1]

[0]: https://lists.sr.ht/~tfardet/nngt-developers/patches/23533
[1]: mailto:tanguyfardet@protonmail.com

✗ #535060 FAILED NNGT/patches/.build.yml https://builds.sr.ht/~tfardet/job/535060