Add pytest runner for test snippets

py_code_object/Pipfile.lock

@@ -0,0 +1,91 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "ce98de5914393363a8cb86a4753b3964caa53a4659a403a3ef357e2086363ef7"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.6"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "aenum": {
+            "hashes": [
+                "sha256:3df9b84cce5dc9ed77c337079f97b66c44c0053eb87d6f4d46b888dc45801e38",
+                "sha256:7a77c205c4bc9d7fe9bd73b3193002d724aebf5909fa0d297534208953891ec8",
+                "sha256:a3208e4b28db3a7b232ff69b934aef2ea1bf27286d9978e1e597d46f490e4687"
+            ],
+            "version": "==2.1.2"
+        },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
+                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
+            ],
+            "version": "==1.1.5"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
+                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
+            ],
+            "version": "==18.1.0"
+        },
+        "bytecode": {
+            "hashes": [
+                "sha256:cc6931151c7f0a542f8cf7619fe1639af3b9529c4678860fa3239397cb0f7de0",
+                "sha256:e464004d4a9eeeca987cb4950dba11b827964b6c90cd331c1f20abd2dab3c962"
+            ],
+            "index": "pypi",
+            "version": "==0.7.0"
+        },
+        "more-itertools": {
+            "hashes": [
+                "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8",
+                "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3",
+                "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0"
+            ],
+            "version": "==4.2.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff",
+                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
+                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5"
+            ],
+            "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.7'",
+            "version": "==0.6.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
+                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
+            ],
+            "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.7'",
+            "version": "==1.5.4"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:0453c8676c2bee6feb0434748b068d5510273a916295fd61d306c4f22fbfd752",
+                "sha256:4b208614ae6d98195430ad6bde03641c78553acee7c83cec2e85d613c0cd383d"
+            ],
+            "index": "pypi",
+            "version": "==3.6.3"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        }
+    },
+    "develop": {}
+}

py_code_object/compile_code.py

@@ -0,0 +1,60 @@
+import bytecode
+import sys
+import json
+import types
+
+
+class CodeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if (isinstance(obj, types.CodeType)):
+            return serialize_code(obj)
+        return json.JSONEncoder.default(self, obj)
+
+def serialize_code(code):
+    c = bytecode.Bytecode().from_code(code).to_concrete_bytecode()
+    return (
+        {
+            "co_consts": consts_to_rust_enum(c.consts),
+            "co_names": c.names,
+            "co_name": c.name,
+            "co_code": parse_co_code_to_str(c),
+            "co_varnames": c.varnames
+        }
+    )
+
+
+def consts_to_rust_enum(consts):
+    def capitalize_first(s):
+        return s[0].upper() + s[1:]
+
+    def const_to_rust_enum(const):
+        if type(const).__name__ == "tuple":
+            return {capitalize_first(str(type(const).__name__)): list(map(const_to_rust_enum, const))}
+        else:
+            return {capitalize_first(str(type(const).__name__)): const}
+    return list(map(const_to_rust_enum, consts))
+
+
+def parse_co_code_to_str(c):
+    return list(
+        map(lambda op: (op.size, op.name, op.arg if op.arg != bytecode.UNSET else None),
+            c)
+    )
+
+
+def compile_to_bytecode(filename, out_file=None):
+    with open(filename, 'rU') as f:
+        code = f.read()
+
+    code = compile(code, filename, "exec")
+
+    print(CodeEncoder().encode(code), file=out_file)
+
+
+def main():
+    filename = sys.argv[1]
+    compile_to_bytecode(filename)
+
+
+if __name__ == "__main__":
+    main()

py_code_object/requirements.txt

@@ -0,0 +1 @@
+byteplay==0.2

py_code_object/src/convert.rs

@@ -0,0 +1,10 @@
+
+// TODO: create a function which takes CPython bytecode and transforms
+// this into RustPython bytecode. This to decouple RustPython from CPython
+// internal bytecode representations.
+
+
+pub fn convert(cpython_bytecode: CPythonByteCode) -> ByteCode {
+    panic!("TODO");
+}
+

py_code_object/test_all.bat

@@ -0,0 +1,11 @@
+::  win bat 
+
+
+cd tests
+
+for %%i in (*.py) do python ../compile_code.py %%i >../bytes/%%i.bytecode
+cd ..
+
+REM cd RustPython
+REM for %%i in (../bytes/*.vbytecode) do echo %%i
+REM cd ..

py_code_object/test_all.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# set -e 
+
+source venv/bin/activate
+#TESTCASE=tests/variables.py
+# TESTCASE=tests/variables.py
+#TESTCASE=tests/minimum.py
+unexpected_count=0
+expected_count=0
+fail_titles=$""
+
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+for TESTCASE in $(find ../tests -name \*.py -print)
+do
+  echo "TEST START: ${TESTCASE}"
+  echo "--------------------------------"
+  FILENAME="$(basename ${TESTCASE})"
+  xfail=false
+  if [ "${FILENAME:0:6}" = "xfail_" ]; then
+    echo "Expected FAIL"
+    xfail=true
+  fi
+
+
+  python compile_code.py $TESTCASE > $TESTCASE.bytecode
+  cd RustPython
+  cargo run ../$TESTCASE.bytecode
+
+
+  if [[ $? -ne 0 ]]; then
+    if [ "${xfail}" = true ]; then
+      echo "== FAIL as expected  =="
+      let expected_count=expected_count+1
+    else
+      printf "${RED}== FAIL, expected PASS ==${NC}\n"
+      let unexpected_count=unexpected_count+1
+      fail_titles=$"${fail_titles}\n${TESTCASE}\t${RED}FAIL${NC} (expected PASS)"
+    fi
+  else
+    if [ "${xfail}" = true ]; then
+      printf "${RED}== PASS, expected FAIL ==${NC}\n"
+      let unexpected_count=unexpected_count+1
+      let unexpected_count=unexpected_count+1
+      fail_titles=$"${fail_titles}\n${TESTCASE}\t${RED}PASS${NC} (expected FAIL)"
+    else
+      echo "== PASS as expected  =="
+      let expected_count=expected_count+1
+    fi
+  fi
+  cd ..
+  echo "--------------------------------"
+
+done
+
+echo "Summary"
+echo "================"
+printf "${RED}${unexpected_count} unexpected${NC}, ${expected_count} expected"
+echo ""
+echo ""
+echo "unexpected results:"
+printf "${fail_titles}"
+echo ""
+echo "================"

py_code_object/test_snippets.py

@@ -0,0 +1,72 @@
+
+# This is a python unittest class automatically populating with all tests
+# in the tests folder.
+
+
+import os
+import unittest
+import glob
+import logging
+import subprocess
+import contextlib
+
+import compile_code
+
+
+logger = logging.getLogger('tests')
+TEST_DIR = os.path.abspath(os.path.join('..', 'tests'))
+CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join('..', 'vm', 'RustPython'))
+
+
+@contextlib.contextmanager
+def pushd(path):
+    old_dir = os.getcwd()
+    os.chdir(path)
+    yield
+    os.chdir(old_dir)
+
+
+def perform_test(filename):
+    logger.info('Running %s', filename)
+    # Step1: Create bytecode file:
+    bytecode_filename = filename + '.bytecode'
+    with open(bytecode_filename, 'w') as f:
+        compile_code.compile_to_bytecode(filename, out_file=f)
+
+    # Step2: run cpython bytecode:
+    with pushd(CPYTHON_RUNNER_DIR):
+        subprocess.check_call(['cargo', 'run', bytecode_filename])
+
+
+def create_test_function(cls, filename):
+    """ Create a test function for a single snippet """
+    core_test_directory, snippet_filename = os.path.split(filename)
+    test_function_name = 'test_' \
+        + os.path.splitext(snippet_filename)[0] \
+        .replace('.', '_').replace('-', '_')
+
+    def test_function(self):
+        perform_test(filename)
+
+    if hasattr(cls, test_function_name):
+        raise ValueError('Duplicate test case {}'.format(test_function_name))
+    setattr(cls, test_function_name, test_function)
+
+
+def populate(cls):
+    """ Decorator function which can populate a unittest.TestCase class """
+    for filename in get_test_files():
+        create_test_function(cls, filename)
+    return cls
+
+
+def get_test_files():
+    """ Retrieve test files """
+    for filename in sorted(glob.iglob(os.path.join(
+            TEST_DIR, '*.py'))):
+        yield os.path.abspath(filename)
+
+
+@populate
+class SampleTestCase(unittest.TestCase):
+    pass