Ask Your Question

Revision history [back]

Cython problem with map-reduce

Below some minimal code:

# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
    S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
    return S.run()

whose compilation reveals the following Cython problem with map-reduce:

┌────────────────────────────────────────────────────────────────────┐
│ SageMath version 8.3, Release Date: 2018-08-03                     │
│ Type "notebook()" for the browser-based notebook interface.        │
│ Type "help()" for help.                                            │
└────────────────────────────────────────────────────────────────────┘
sage: %attach SAGE/test.spyx
Compiling ./SAGE/test.spyx...
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-1-506a10c79ce0> in <module>()
----> 1 get_ipython().magic(u'attach SAGE/test.spyx')

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in magic(self, arg_s)
   2158         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2159         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2160         return self.run_line_magic(magic_name, magic_arg_s)
   2161
   2162     #-------------------------------------------------------------------------

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_line_magic(self, magic_name, line)
   2079                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2080             with self.builtin_trap:
-> 2081                 result = fn(*args,**kwargs)
   2082             return result
   2083

<decorator-gen-115> in attach(self, s)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189
    190         if callable(arg):

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/ipython_extension.pyc in attach(self, s)
    156             sage: shell.quit()
    157         """
--> 158         return self.shell.ex(load_wrap(s, attach=True))
    159
    160     @line_magic

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in ex(self, cmd)
   2421         """Execute a normal python statement in user namespace."""
   2422         with self.builtin_trap:
-> 2423             exec(cmd, self.user_global_ns, self.user_ns)
   2424
   2425     def ev(self, expr):

<string> in <module>()

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load(filename, globals, attach)
    265         if attach:
    266             add_attached_file(fpath)
--> 267         exec(load_cython(fpath), globals)
    268     elif ext == '.f' or ext == '.f90':
    269         from sage.misc.inline_fortran import fortran

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load_cython(name)
     65     """
     66     from sage.misc.cython import cython
---> 67     mod, dir = cython(name, compile_message=True, use_cache=True)
     68     import sys
     69     sys.path.append(dir)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/misc/cython.pyc in cython(filename, verbose, compile_message, use_cache, create_local_c_file, annotate, sage_namespace, create_local_so_file)
    636                         You can fix your code by adding "from {} cimport {}".
    637                         """.format(pxd, name))
--> 638         raise RuntimeError(cython_messages.strip())
    639
    640     if verbose >= 0:

RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
     ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:5:6: closures inside cpdef functions not yet supported

Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
        S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
                                           ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:6:44: Compiler crash in CreateClosureClasses

ModuleNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:3:0)
StatListNode.stats[1] = CFuncDefNode(_home_sage_SAGE_test_spyx_0.pyx:5:6,
    args = [...]/1,
    modifiers = [...]/0,
    needs_closure = True,
    overridable = 1,
    visibility = u'private')
CFuncDefNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:6:1,
    is_terminator = True)
StatListNode.stats[0] = SingleAssignmentNode(_home_sage_SAGE_test_spyx_0.pyx:6:19)
SingleAssignmentNode.rhs = GeneralCallNode(_home_sage_SAGE_test_spyx_0.pyx:6:19,
    is_temp = 1,
    result_is_used = True,
    use_managed_ref = True)
GeneralCallNode.keyword_args = DictNode(_home_sage_SAGE_test_spyx_0.pyx:6:20,
    is_dict_literal = True,
    is_temp = 1,
    obj_conversion_errors = [...]/0,
    reject_duplicates = True,
    result_is_used = True,
    use_managed_ref = True)
DictNode.key_value_pairs[1] = DictItemNode(_home_sage_SAGE_test_spyx_0.pyx:6:33,
    result_is_used = True,
    use_managed_ref = True)
DictItemNode.value = LambdaNode(_home_sage_SAGE_test_spyx_0.pyx:6:44,
    args = [...]/1,
    binding = True,
    is_temp = 1,
    lambda_name = 'lambda',
    name = u'<lambda>',
    needs_closure = True,
    needs_self_code = True,
    pymethdef_cname = u'__pyx_mdef_27_home_sage_SAGE_test_spyx_0_4test_lambda',
    result_is_used = True,
    use_managed_ref = True)

Compiler crash traceback from this point on:
  File "Cython/Compiler/Visitor.py", line 180, in Cython.Compiler.Visitor.TreeVisitor._visit
    return handler_method(obj)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2764, in visit_LambdaNode
    self.create_class_from_scope(node.def_node, self.module_scope, node)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2713, in create_class_from_scope
    func_scope.scope_class = cscope.scope_class
AttributeError: 'ClosureScope' object has no attribute 'scope_class'

Cython problem with map-reduce

Below some minimal code: code using map-reduce:

# %attach SAGE/test.spyx
 from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
    S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
    return S.run()

whose compilation reveals the following a Cython problem with map-reduce:Error compiling Cython file , AttributeError: 'ClosureScope' object has no attribute 'scope_class', see below:

┌────────────────────────────────────────────────────────────────────┐
│ SageMath version 8.3, Release Date: 2018-08-03                     │
│ Type "notebook()" for the browser-based notebook interface.        │
│ Type "help()" for help.                                            │
└────────────────────────────────────────────────────────────────────┘
sage: %attach SAGE/test.spyx
Compiling ./SAGE/test.spyx...
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-1-506a10c79ce0> in <module>()
----> 1 get_ipython().magic(u'attach SAGE/test.spyx')

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in magic(self, arg_s)
   2158         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2159         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2160         return self.run_line_magic(magic_name, magic_arg_s)
   2161
   2162     #-------------------------------------------------------------------------

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_line_magic(self, magic_name, line)
   2079                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2080             with self.builtin_trap:
-> 2081                 result = fn(*args,**kwargs)
   2082             return result
   2083

<decorator-gen-115> in attach(self, s)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189
    190         if callable(arg):

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/ipython_extension.pyc in attach(self, s)
    156             sage: shell.quit()
    157         """
--> 158         return self.shell.ex(load_wrap(s, attach=True))
    159
    160     @line_magic

/opt/sagemath-8.3/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in ex(self, cmd)
   2421         """Execute a normal python statement in user namespace."""
   2422         with self.builtin_trap:
-> 2423             exec(cmd, self.user_global_ns, self.user_ns)
   2424
   2425     def ev(self, expr):

<string> in <module>()

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load(filename, globals, attach)
    265         if attach:
    266             add_attached_file(fpath)
--> 267         exec(load_cython(fpath), globals)
    268     elif ext == '.f' or ext == '.f90':
    269         from sage.misc.inline_fortran import fortran

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/repl/load.pyc in load_cython(name)
     65     """
     66     from sage.misc.cython import cython
---> 67     mod, dir = cython(name, compile_message=True, use_cache=True)
     68     import sys
     69     sys.path.append(dir)

/opt/sagemath-8.3/local/lib/python2.7/site-packages/sage/misc/cython.pyc in cython(filename, verbose, compile_message, use_cache, create_local_c_file, annotate, sage_namespace, create_local_so_file)
    636                         You can fix your code by adding "from {} cimport {}".
    637                         """.format(pxd, name))
--> 638         raise RuntimeError(cython_messages.strip())
    639
    640     if verbose >= 0:

RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
     ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:5:6: closures inside cpdef functions not yet supported

Error compiling Cython file:
------------------------------------------------------------
...
# %attach SAGE/test.spyx

from sage.parallel.map_reduce import RESetMapReduce

cpdef test(int n):
        S = RESetMapReduce(roots = [[]],children = lambda l: [l+[0], l+[1]] if len(l) <= n else [],map_function = lambda x : 1,reduce_function = lambda x,y: x+y,reduce_init = 0)
                                           ^
------------------------------------------------------------

_home_sage_SAGE_test_spyx_0.pyx:6:44: Compiler crash in CreateClosureClasses

ModuleNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:3:0)
StatListNode.stats[1] = CFuncDefNode(_home_sage_SAGE_test_spyx_0.pyx:5:6,
    args = [...]/1,
    modifiers = [...]/0,
    needs_closure = True,
    overridable = 1,
    visibility = u'private')
CFuncDefNode.body = StatListNode(_home_sage_SAGE_test_spyx_0.pyx:6:1,
    is_terminator = True)
StatListNode.stats[0] = SingleAssignmentNode(_home_sage_SAGE_test_spyx_0.pyx:6:19)
SingleAssignmentNode.rhs = GeneralCallNode(_home_sage_SAGE_test_spyx_0.pyx:6:19,
    is_temp = 1,
    result_is_used = True,
    use_managed_ref = True)
GeneralCallNode.keyword_args = DictNode(_home_sage_SAGE_test_spyx_0.pyx:6:20,
    is_dict_literal = True,
    is_temp = 1,
    obj_conversion_errors = [...]/0,
    reject_duplicates = True,
    result_is_used = True,
    use_managed_ref = True)
DictNode.key_value_pairs[1] = DictItemNode(_home_sage_SAGE_test_spyx_0.pyx:6:33,
    result_is_used = True,
    use_managed_ref = True)
DictItemNode.value = LambdaNode(_home_sage_SAGE_test_spyx_0.pyx:6:44,
    args = [...]/1,
    binding = True,
    is_temp = 1,
    lambda_name = 'lambda',
    name = u'<lambda>',
    needs_closure = True,
    needs_self_code = True,
    pymethdef_cname = u'__pyx_mdef_27_home_sage_SAGE_test_spyx_0_4test_lambda',
    result_is_used = True,
    use_managed_ref = True)

Compiler crash traceback from this point on:
  File "Cython/Compiler/Visitor.py", line 180, in Cython.Compiler.Visitor.TreeVisitor._visit
    return handler_method(obj)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2764, in visit_LambdaNode
    self.create_class_from_scope(node.def_node, self.module_scope, node)
  File "/opt/sagemath-8.3/local/lib/python2.7/site-packages/Cython/Compiler/ParseTreeTransforms.py", line 2713, in create_class_from_scope
    func_scope.scope_class = cscope.scope_class
AttributeError: 'ClosureScope' object has no attribute 'scope_class'