refine codes in runtime (#2765)

Author: sneaxiy

python/runtime/alps/train.py

@@ -16,25 +16,17 @@
 
 import os
 
-import tensorflow as tf
 from alps.client.base import run_experiment, submit_experiment
-from alps.conf.closure import Closure
-from alps.framework.column.column import (DenseColumn, GroupedSparseColumn,
-                                          SparseColumn)
 from alps.framework.engine import (KubemakerEngine, LocalEngine, ResourceConf,
                                    YarnEngine)
-from alps.framework.experiment import (EstimatorBuilder, EvalConf, Experiment,
-                                       RuntimeConf, TrainConf)
+from alps.framework.experiment import (EvalConf, Experiment, RuntimeConf,
+                                       TrainConf)
 from alps.framework.exporter import ExportStrategy
 from alps.framework.exporter.arks_exporter import ArksExporter
 from alps.framework.exporter.base import Goal, MetricComparator
-from alps.framework.train.training import build_run_config
 from alps.io import DatasetX
-from alps.io.base import FeatureMap, OdpsConf
+from alps.io.base import FeatureMap
 from alps.io.reader.odps_reader import OdpsReader
-from alps.util.remote_module import RemoteModule
-from tensorflow.python.saved_model.signature_constants import \
-    DEFAULT_SERVING_SIGNATURE_DEF_KEY
 
 # for debug usage.
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
@@ -109,7 +101,7 @@ def train(model_builder,
     else:
         runtime_conf = None
 
-    if max_steps == None:
+    if max_steps is None:
         keep_checkpoint_max = 100
     else:
         keep_checkpoint_max = int(max_steps / 100)

python/runtime/alps/train_example.py

@@ -15,9 +15,7 @@
 import shutil
 
 import tensorflow as tf
-from alps.framework.column.column import (DenseColumn, GroupedSparseColumn,
-                                          SparseColumn)
-from alps.framework.engine import LocalEngine
+from alps.framework.column.column import DenseColumn, SparseColumn
 from alps.framework.experiment import EstimatorBuilder
 from alps.io.base import OdpsConf
 from runtime.alps.train import train
@@ -65,7 +63,8 @@ def _build(self, experiment, run_config):
     odps_conf = OdpsConf(
         accessid=os.getenv("SQLFLOW_TEST_DB_MAXCOMPUTE_AK"),
         accesskey=os.getenv("SQLFLOW_TEST_DB_MAXCOMPUTE_SK"),
-        # endpoint should looks like: "https://service.cn.maxcompute.aliyun.com/api"
+        # endpoint should looks like:
+        # "https://service.cn.maxcompute.aliyun.com/api"
         endpoint=os.getenv("SQLFLOW_TEST_DB_MAXCOMPUTE_ENDPOINT"),
         project=odps_project)
     features = [

python/runtime/alps/train_test.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import subprocess
 import unittest
 from unittest import TestCase
@@ -20,8 +19,11 @@
 
 
 class TestALPSTrain(TestCase):
-    '''NOTE: we must test tensorflow training and predicting in separated processes, or
-    TensorFlow will raise error "Graph is finalized.'''
+    '''
+    NOTE: we must test tensorflow training and predicting
+    in separated processes, or TensorFlow will raise error
+    "Graph is finalized.
+    '''
     @unittest.skipUnless(testing.get_submitter() == "alps",
                          "skip non alps tests")
     def test_train(self):
@@ -33,7 +35,7 @@ def test_train(self):
                                  env={"PYTHONPATH": "python"},
                                  check=True)
             self.assertEqual(ret.returncode, 0)
-        except:
+        except:  # noqa: E722
             self.fail("%s" % ret.stderr)
 
 

python/runtime/local/xgboost/__init__.py

@@ -11,4 +11,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from runtime.local.xgboost.train import train
+from runtime.local.xgboost.train import train  # noqa: F401

python/runtime/local/xgboost/train_test.py

@@ -21,8 +21,8 @@
 from runtime.xgboost.dataset import xgb_dataset
 
 # iris dataset features meta
-# TODO(yancey1989): implement runtime.feature_derivation API to generate the following
-# feature metas
+# TODO(yancey1989): implement runtime.feature_derivation API to generate
+# the following feature metas
 feature_metas = {
     "sepal_length": {
         "feature_name": "sepal_length",

python/runtime/optimize/__init__.py

@@ -11,11 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from runtime.optimize.optflow import run_optimize_on_optflow
+from runtime.optimize.optflow import run_optimize_on_optflow  # noqa: F401
 
 # Step images which submit job to OptFlow may not require pyomo module
 try:
-    from runtime.optimize.local import (generate_model_with_data_frame,
-                                        run_optimize_locally)
-except:
+    from runtime.optimize.local import generate_model_with_data_frame  # noqa: F401, E501
+    from runtime.optimize.local import run_optimize_locally  # noqa: F401, E501
+except:  # noqa: E722
     pass

python/runtime/optimize/local.py

@@ -19,14 +19,8 @@
 import runtime.db as db
 import runtime.verifier as verifier
 import six
-from pyomo.environ import (Binary, Integers, NegativeIntegers, NegativeReals,
-                           NonNegativeIntegers, NonNegativeReals,
-                           NonPositiveIntegers, NonPositiveReals,
-                           PositiveIntegers, PositiveReals, Reals, maximize,
-                           minimize)
 from runtime.optimize.model_generation import (
-    generate_objective_and_constraint_expression,
-    generate_unique_result_value_name)
+    generate_objective_and_constraint_expr, generate_unique_result_value_name)
 
 # FIXME(sneaxiy): do not know why Pyomo requires that the data frame must be
 # a global variable
@@ -47,25 +41,35 @@ def generate_model_with_data_frame(data_frame, variables, variable_type,
         result_value_name (str): the result value name to be optimized.
         objective (list[str]): the objective string token list.
         direction (str): "maximize" or "minimize".
-        constraints (dict): the constraint expression containing the token list and GROUP BY column name.
+        constraints (dict): the constraint expression containing the token list
+            and GROUP BY column name.
 
     Returns:
         A Pyomo ConcreteModel.
     """
     direction = direction.lower()
-    if direction not in ['maximize', 'minimize']:
+    if direction == 'maximize':
+        direction = pyomo_env.maximize
+    elif direction == 'minimize':
+        direction = pyomo_env.minimize
+    else:
         raise ValueError("direction must be one of 'maximize' or 'minimize'")
 
+    if not hasattr(pyomo_env, variable_type):
+        raise ValueError("cannot find variable type %s" % variable_type)
+
+    variable_type = getattr(pyomo_env, variable_type)
+
     model = pyomo_env.ConcreteModel()
     var_num = len(data_frame)
-    model.x = pyomo_env.Var(list(range(var_num)), within=eval(variable_type))
+    model.x = pyomo_env.Var(list(range(var_num)), within=variable_type)
 
     columns = data_frame.columns
 
     variable_str = "model.x"
     data_str = "DATA_FRAME"
 
-    obj_expr, c_exprs = generate_objective_and_constraint_expression(
+    obj_expr, c_exprs = generate_objective_and_constraint_expr(
         columns=columns,
         objective=objective,
         constraints=constraints,
@@ -79,21 +83,23 @@ def generate_model_with_data_frame(data_frame, variables, variable_type,
         global DATA_FRAME
         DATA_FRAME = data_frame
         obj_func = eval("lambda model: %s" % obj_expr)
-        model.objective = pyomo_env.Objective(rule=obj_func,
-                                              sense=eval(direction))
+        model.objective = pyomo_env.Objective(rule=obj_func, sense=direction)
 
         for i, (expr, for_range, iter_vars) in enumerate(c_exprs):
             attr_name = "constraint_%d" % i
 
             if for_range:
-                assert iter_vars, "for_range and iter_vars must be both non-empty"
+                assert iter_vars, "for_range and iter_vars must be " \
+                                  "both non-empty"
                 setattr(model, attr_name, pyomo_env.ConstraintList())
                 constraint_list = getattr(model, attr_name)
-                add_constraint_str = "lambda model, constraint_list: [constraint_list.add(%s) for %s in %s]" % (
-                    expr, ",".join(iter_vars), for_range)
+                template = "lambda model, constraint_list: [constraint_list.add(%s) for %s in %s]"  # noqa: E501
+                add_constraint_str = template % (expr, ",".join(iter_vars),
+                                                 for_range)
                 eval(add_constraint_str)(model, constraint_list)
             else:
-                assert not iter_vars, "for_range and iter_vars must be both empty"
+                assert not iter_vars, \
+                    "for_range and iter_vars must be both empty"
                 func = eval('lambda model: %s' % expr)
                 constraint = pyomo_env.Constraint(rule=func)
                 setattr(model, attr_name, constraint)
@@ -137,8 +143,8 @@ def solve_model(model, solver):
         if pyomo_dtype is None:
             pyomo_dtype = type(model.x[idx])
 
-        assert pyomo_dtype == type(
-            model.x[idx]), "all variables must be of the same data type"
+        assert isinstance(model.x[idx], pyomo_dtype), \
+            "all variables must be of the same data type"
 
     if has_error:
         msg = 'Solve model error. Termination condition: {}.'\
@@ -187,7 +193,8 @@ def save_solved_result_in_db(solved_result, data_frame, variables,
     Save the solved result of the Pyomo model into the database.
 
     Args:
-        solved_result (numpy.ndarray): a numpy array which indicates the solved result.
+        solved_result (numpy.ndarray): a numpy array which indicates
+            the solved result.
         data_frame (panda.DataFrame): the input table data.
         variables (list[str]): the variable names to be optimized.
         result_value_name (str): the result value name to be optimized.
@@ -244,7 +251,8 @@ def run_optimize_locally(datasource, select, variables, variable_type,
         result_value_name (str): the result value name to be optimized.
         objective (list[str]): the objective string token list.
         direction (str): "maximize" or "minimize".
-        constraints (dict): the constraint expression containing the token list and GROUP BY column name.
+        constraints (dict): the constraint expression containing the token list
+            and GROUP BY column name.
         solver (str): the solver used to solve the model.
         result_table (str): the table name to save the solved results.