Ask Your Question
0

Cython problem with map-reduce

asked 2019-06-06 15:51:24 -0500

updated 2019-06-06 16:09:39 -0500

Below some minimal code using map-reduce:

# %attach SAGE/test.spyx
from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
    S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
    return S.run()

whose compilation reveals a Cython problem Error compiling Cython file , AttributeError: 'ClosureScope' object has no attribute 'scope_class', see below:

┌────────────────────────────────────────────────────────────────────┐
│ SageMath version 8.3, Release Date: 2018-08-03                     │
│ Type "notebook()" for the browser-based notebook interface.        │
│ Type "help()" for help.                                            │
└────────────────────────────────────────────────────────────────────┘
sage: %attach SAGE/test.spyx
Compiling ./SAGE/test.spyx...
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-1-506a10c79ce0> in <module>()
----> 1 get_ipython().magic(u'attach SAGE/test.spyx')

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in magic(self, arg_s)
   2158         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2159         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2160         return self.run_line_magic(magic_name, magic_arg_s)
   2161
   2162     #-------------------------------------------------------------------------

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_line_magic(self, magic_name, line)
   2079                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2080             with self.builtin_trap:
-> 2081                 result = fn(*args,**kwargs)
   2082             return result
   2083

<decorator-gen-115> in attach(self, s)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189
    190         if callable(arg):

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/ipython_extension.pyc in attach(self, s)
    156             sage: shell.quit()
    157         """
--> 158         return self.shell.ex(load_wrap(s, attach=True))
    159
    160     @line_magic

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in ex(self, cmd)
   2421         """Execute a normal python statement in user namespace."""
   2422         with self.builtin_trap:
-> 2423             exec(cmd, self.user_global_ns, self.user_ns)
   2424
   2425     def ev(self, expr):

<string> in <module>()

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load(filename, globals, attach)
    265         if attach:
    266             add_attached_file(fpath)
--> 267         exec(load_cython(fpath), globals)
    268     elif ext == '.f' or ext == '.f90':
    269         from sage.misc.inline_fortran import fortran

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load_cython(name)
     65     """
     66     from sage.misc.cython import cython
---> 67     mod, dir = cython(name, compile_message=True, use_cache=True)
     68     import sys
     69     sys.path.append(dir)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/misc/cython.pyc in cython(filename, verbose, compile_message, use_cache, create_local_c_file, annotate, sage_namespace, create_local_so_file)
    636                         You can fix your code by adding "from {} cimport {}".
    637                         """.format(pxd, name))
--> 638         raise RuntimeError(cython_messages.strip())
    639
    640     if verbose >= 0:

RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
     ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:5:6: closures inside cpdef functions not yet supported

Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
        S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
                                           ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:6:44: Compiler crash in CreateClosureClasses

ModuleNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:3:0)
StatListNode.stats[1] = CFuncDefNode(_home_sage_SAGE_test_spyx_0.pyx:5:6,
    args = [...]/1,
    modifiers = [...]/0,
    needs_closure = True,
    overridable = 1,
    visibility = u'private')
CFuncDefNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:6:1,
    is_terminator = True)
StatListNode.stats[0] = SingleAssignmentNode(_home_sage_SAGE_test_spyx_0.pyx:6:19)
SingleAssignmentNode.rhs = GeneralCallNode(_home_sage_SAGE_test_spyx_0.pyx:6:19,
    is_temp = 1,
    result_is_used = True,
    use_managed_ref = True)
GeneralCallNode.keyword_args = DictNode(_home_sage_SAGE_test_spyx_0.pyx:6:20,
    is_dict_literal = True,
    is_temp = 1,
    obj_conversion_errors = [...]/0,
    reject_duplicates = True,
    result_is_used = True,
    use_managed_ref = True)
DictNode.key_value_pairs[1] = DictItemNode(_home_sage_SAGE_test_spyx_0.pyx:6:33,
    result_is_used = True,
    use_managed_ref = True)
DictItemNode.value = LambdaNode(_home_sage_SAGE_test_spyx_0.pyx:6:44,
    args = [...]/1,
    binding = True,
    is_temp = 1,
    lambda_name = 'lambda',
    name = u'<lambda>',
    needs_closure = True,
    needs_self_code = True,
    pymethdef_cname = u'__pyx_mdef_27_home_sage_SAGE_test_spyx_0_4test_lambda',
    result_is_used = True,
    use_managed_ref = True)

Compiler crash traceback from this point on:
  File "Cython/Compiler/Visitor.py", line 180, in Cython.Compiler.Visitor.TreeVisitor._visit
    return handler_method(obj)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2764, in visit_LambdaNode
    self.create_class_from_scope(node.def_node, self.module_scope, node)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2713, in create_class_from_scope
    func_scope.scope_class = cscope.scope_class
AttributeError: 'ClosureScope' object has no attribute 'scope_class'
edit retag flag offensive close merge delete

1 answer

Sort by » oldest newest most voted
2

answered 2019-06-07 02:59:21 -0500

Hivert gravatar image

This has nothing to do with MapReduce. Cython cannot deal with definition of closure (here: lambda function) inside a c(p)def function. If you want to use MapReduce with Cython code, you should not use the short one line interface of map reduce but rely on the programmatic interface using inheritance (see the source code of RESetMPExample for an example).

edit flag offensive delete link more

Your Answer

Please start posting anonymously - your entry will be published after you log in or create a new account.

Add Answer

Question Tools

1 follower

Stats

Asked: 2019-06-06 15:51:24 -0500

Seen: 92 times

Last updated: Jun 07